00010000:00000010:0.1:1786172076.996940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f858800. 00010000:00000010:0.1:1786172077.034298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09311400. 00000080:00000010:0.1:1786172077.227471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e258c6060. 00000080:00000010:0.1:1786172077.227483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e258c6c20. 00010000:00000010:0.1:1786172077.227497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261af600. 00010000:00000010:0.1:1786172077.393770:0:177483:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e12ee2d00. 00010000:00000010:0.1:1786172077.393785:0:177483:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f859000. 00000080:00000010:0.1:1786172077.606541:0:177483:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed19a0. 00010000:00000010:0.1:1786172077.859914:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:0.1:1786172077.859922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e092f5a00. 00010000:00000010:0.1:1786172077.907406:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d1ddf00. 00010000:00000010:0.1:1786172077.907420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09310800. 00000080:00000010:0.1:1786172078.919023:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e10b58b20. 00010000:00000010:0.1:1786172078.959517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f859200. 00010000:00000010:0.1:1786172079.082138:0:177483:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f859200. 00010000:00000010:0.1:1786172079.111385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f859200. 00000080:00000010:0.1:1786172079.522407:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e258c6440. 00010000:00000010:0.1:1786172079.608426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09312c00. 00010000:00000010:0.1:1786172079.708388:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09312200. 00010000:00000010:0.1:1786172079.947408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024e800. 00010000:00000010:0.1:1786172079.963797:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09312c00. 00010000:00000010:0.1:1786172080.236685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0868c300. 00010000:00000010:0.1:1786172080.236697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e092f4200. 00010000:00000010:0.1:1786172080.250504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:0.1:1786172080.332402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09312000. 00010000:00000010:0.1:1786172080.548515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0868c000. 00010000:00000010:0.1:1786172080.548529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024f600. 00010000:00000010:0.1:1786172080.838130:0:177511:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31ec0000. 00010000:00000010:0.1:1786172080.838142:0:177511:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d166a00. 00000080:00000010:0.1:1786172081.050719:0:177511:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed14c0. 00000080:00000010:0.1:1786172082.328059:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117768a0. 00010000:00000010:0.1:1786172083.188574:0:2386:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e32924f00. 00010000:00000010:0.1:1786172083.188592:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d167400. 00000080:00000010:0.1:1786172083.238382:0:177511:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e10b58820. 00010000:00000010:0.1:1786172083.354399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024cc00. 00010000:00000010:0.1:1786172083.826391:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d167400. 00010000:00000010:0.1:1786172083.877401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08c43900. 00010000:00000010:0.1:1786172083.877408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d266a00. 00010000:00000010:0.1:1786172084.084437:0:40394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d0ad000. 00010000:00000010:0.1:1786172084.165379:0:2394:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08c43200. 00010000:00000010:0.1:1786172084.165386:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d267000. 00010000:00000010:0.1:1786172084.254376:0:165139:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12c99200. 00000080:00000010:0.1:1786172084.280884:0:2385:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1ce0. 00010000:00000010:0.1:1786172084.601404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e498700. 00010000:00000010:0.1:1786172084.601420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12c9be00. 00010000:00000010:0.1:1786172084.638382:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12c98200. 00010000:00000010:0.1:1786172084.694440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e32924c00. 00010000:00000010:0.1:1786172084.694448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d0ad200. 00010000:00000010:0.1:1786172084.911442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12c98800. 00010000:00000010:0.1:1786172086.026612:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09311200. 00010000:00000010:0.1:1786172086.424396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31295800. 00000080:00000010:0.1:1786172086.943504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31d2e240. 00000080:00000010:0.1:1786172087.055457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776c80. 00010000:00000010:0.1:1786172087.127437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31295e00. 00000080:00000010:0.1:1786172087.356401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31d2ed00. 00000080:00000010:0.1:1786172087.356409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31d2ea80. 00010000:00000010:0.1:1786172087.356424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e316e9600. 00010000:00000010:0.1:1786172087.356428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2da00. 00000080:00000010:0.1:1786172087.356431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776a40. 00010000:00000010:0.1:1786172087.356435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e316e9800. 00010000:00000010:0.1:1786172087.356438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2c400. 00010000:00000010:0.1:1786172087.728381:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d82e400. 00000080:00000010:0.1:1786172088.234414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1513a960. 00010000:00000010:0.1:1786172088.410499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2c000. 00010000:00000010:0.1:1786172088.431470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2c000. 00010000:00000010:0.1:1786172088.527851:0:177546:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2d400. 00010000:00000010:0.1:1786172088.832435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31b3d400. 00000080:00000010:0.1:1786172088.857596:0:176663:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e055b5400. 00010000:00000010:0.1:1786172088.940403:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d165000. 00010000:00000010:0.1:1786172088.940410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2a800. 00010000:00000010:0.1:1786172089.269856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2aa00. 00000080:00000010:0.1:1786172089.320458:0:176663:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31d2e0e0. 00000080:00000010:0.1:1786172089.320467:0:176663:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1f60. 00010000:00000010:0.1:1786172089.350526:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e316e9b00. 00010000:00000010:0.1:1786172089.350532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2da00. 00000080:00000010:0.1:1786172089.373542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776da0. 00010000:00000010:0.1:1786172089.460178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d165d00. 00010000:00000010:0.1:1786172089.460191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2b600. 00000080:00000010:0.1:1786172089.498817:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1560. 00010000:00000010:0.1:1786172089.510466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2c800. 00000080:00000010:0.1:1786172089.731421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e3205ca60. 00010000:00000010:0.1:1786172089.766431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e316e9200. 00010000:00000010:0.1:1786172089.766443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2c800. 00000080:00000010:0.1:1786172089.822677:0:2394:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776a40. 00010000:00000010:0.1:1786172089.831384:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e316e9200. 00010000:00000010:0.1:1786172089.831390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2d800. 00010000:00000010:0.1:1786172089.831393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527f800. 00010000:00000010:0.1:1786172089.854478:0:177568:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527ea00. 00010000:00000010:0.1:1786172090.131654:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10a9c200. 00010000:00000010:0.1:1786172090.131663:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d82e400. 00000080:00000010:0.1:1786172090.153862:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1ca0. 00010000:00000010:0.1:1786172090.647122:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d82e200. 00000080:00000010:0.1:1786172090.937415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12d41d80. 00010000:00000010:0.1:1786172091.252132:0:177568:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e189dc000. 00010000:00000010:0.1:1786172091.252142:0:177568:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527fa00. 00000080:00000010:0.1:1786172091.984403:0:177568:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e0921f120. 00010000:00000010:0.1:1786172092.469921:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10a9cd00. 00010000:00000010:0.1:1786172092.469933:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39cbde00. 00000080:00000010:0.1:1786172092.500252:0:176663:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7820. 00010000:00000010:0.1:1786172092.517680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31d88a00. 00010000:00000010:0.1:1786172092.589866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fada200. 00000080:00000010:0.1:1786172092.601420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117763c0. 00010000:00000010:0.1:1786172092.909223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadae00. 00010000:00000010:0.1:1786172092.923902:0:177568:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d165d00. 00010000:00000010:0.1:1786172092.923910:0:177568:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2aa00. 00010000:00000010:0.1:1786172092.953544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadae00. 00010000:00000010:0.1:1786172093.141423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31d89000. 00010000:00000010:0.1:1786172093.225104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadae00. 00010000:00000010:0.1:1786172093.245404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadae00. 00010000:00000010:0.1:1786172093.263392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadae00. 00010000:00000010:0.1:1786172093.454478:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e316e9700. 00010000:00000010:0.1:1786172093.496390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadac00. 00010000:00000010:0.1:1786172093.583557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39cbca00. 00000080:00000010:0.1:1786172093.812668:0:177568:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7c00. 00010000:00000010:0.1:1786172094.096401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d980e00. 00010000:00000010:0.1:1786172094.096408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527c400. 00000080:00000010:0.1:1786172094.387643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e0921f300. 00000080:00000010:0.1:1786172094.889495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e0921f7a0. 00010000:00000010:0.1:1786172095.025383:0:177593:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e093bbe00. 00010000:00000010:0.1:1786172095.025389:0:177593:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d83200. 00000080:00000010:0.1:1786172095.717897:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7ce0. 00010000:00000010:0.1:1786172095.954465:0:176663:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f976a00. 00000080:00000010:0.1:1786172095.954470:0:176663:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e0921f3c0. 00010000:00000010:0.1:1786172095.959467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e3cee00. 00010000:00000010:0.1:1786172095.959473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f34a00. 00000080:00000010:0.1:1786172095.959478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776420. 00000080:00000010:0.1:1786172095.964558:0:177593:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776200. 00000080:00000010:0.1:1786172095.996663:0:177593:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776d00. 00010000:00000010:0.1:1786172096.124504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d83800. 00000080:00000010:0.1:1786172096.296440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776000. 00000080:00000010:0.1:1786172096.314135:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776b00. 00010000:00000010:0.1:1786172096.318479:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e3178aa00. 00010000:00000010:0.1:1786172096.318484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d83200. 00000080:00000010:0.1:1786172096.463398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e0921f8c0. 00010000:00000010:0.1:1786172096.463408:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e093bbd00. 00010000:00000010:0.1:1786172096.463412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f976800. 00000080:00000010:0.1:1786172096.463414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776d80. 00000080:00000010:0.1:1786172096.512313:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7920. 00000080:00000010:0.1:1786172096.531409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7660. 00000080:00000010:0.1:1786172096.531423:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776d80. 00010000:00000010:0.1:1786172096.572402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d82a00. 00000080:00000010:0.1:1786172096.584411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776d80. 00010000:00000010:0.1:1786172096.640354:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e3178a500. 00010000:00000010:0.1:1786172096.640381:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d82e00. 00000080:00000010:0.1:1786172096.703391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7000. 00010000:00000010:0.1:1786172096.766777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31072400. 00000080:00000010:0.1:1786172096.905503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7300. 00010000:00000010:0.1:1786172096.916490:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68ba00. 00010000:00000010:0.1:1786172096.916498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d83200. 00000080:00000010:0.1:1786172096.927080:0:176663:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117764e0. 00000080:00000010:0.1:1786172096.927094:0:176663:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776640. 00010000:00000010:0.1:1786172096.969426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d83400. 00000080:00000010:0.1:1786172097.047053:0:177610:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776fe0. 00000080:00000010:0.1:1786172097.121431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1200. 00000080:00000010:0.1:1786172097.121442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1be0. 00010000:00000010:0.1:1786172097.181662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:0.1:1786172097.195136:0:176663:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e3178a100. 00010000:00000010:0.1:1786172097.195146:0:176663:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d82200. 00000080:00000010:0.1:1786172097.230811:0:152385:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776dc0. 00010000:00000010:0.1:1786172097.436824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d83a00. 00010000:00000010:0.1:1786172097.472335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31072c00. 00010000:00000010:0.1:1786172097.711648:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00000080:00000010:0.1:1786172097.746386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1ae0. 00010000:00000010:0.1:1786172097.755545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e093bb200. 00010000:00000010:0.1:1786172097.755554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00000080:00000010:0.1:1786172097.761877:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e0921f7c0. 00000080:00000010:0.1:1786172097.797491:0:177593:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776de0. 00000080:00000010:0.1:1786172097.813516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776300. 00000080:00000010:0.1:1786172097.828382:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776fc0. 00010000:00000010:0.1:1786172097.836955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb600. 00010000:00000010:0.1:1786172098.071452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31789000. 00000080:00000010:0.1:1786172098.088613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7000. 00010000:00000010:0.1:1786172098.098400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb800. 00000080:00000010:0.1:1786172098.098418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117760c0. 00000080:00000010:0.1:1786172098.114479:0:176663:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117764e0. 00010000:00000010:0.1:1786172098.121694:0:176663:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca200. 00000080:00000010:0.1:1786172098.144847:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776640. 00010000:00000010:0.1:1786172098.147631:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb600. 00000080:00000010:0.1:1786172099.133722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117766a0. 00010000:00000010:0.1:1786172099.177443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e3178a700. 00010000:00000010:0.1:1786172099.177456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca200. 00010000:00000010:0.1:1786172099.190121:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e3178a400. 00000080:00000010:0.1:1786172099.497243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e76a0. 00000080:00000010:0.1:1786172099.819528:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed19c0. 00000080:00000010:0.1:1786172099.819535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1660. 00010000:00000010:0.1:1786172099.898402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12c65800. 00000080:00000010:0.1:1786172099.898410:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed18a0. 00010000:00000010:0.1:1786172099.967430:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02f48600. 00010000:00000010:0.1:1786172099.967442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39cbfc00. 00010000:00000010:0.1:1786172100.385553:0:176663:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31292c00. 00000080:00000010:0.1:1786172100.433782:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e0921fe60. 00000080:00000010:0.1:1786172100.714474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776040. 00010000:00000010:0.1:1786172100.967383:0:177647:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e093b8000. 00010000:00000010:0.1:1786172100.967391:0:177647:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39cbf200. 00010000:00000010:0.1:1786172101.792128:0:1028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31292000. 00000080:00000010:0.1:1786172102.036242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7ce0. 00000080:00000010:0.1:1786172102.104171:0:177647:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776d80. 00010000:00000010:0.1:1786172102.137998:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02fb6200. 00000080:00000010:0.1:1786172102.183430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7920. 00010000:00000010:0.1:1786172102.724448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39420400. 00000080:00000010:0.1:1786172102.774421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117762c0. 00010000:00000010:0.1:1786172102.779523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39420200. 00010000:00000010:0.1:1786172102.782425:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9200. 00010000:00000010:0.1:1786172102.782451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26320e00. 00010000:00000010:0.1:1786172102.794394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31292000. 00000080:00000010:0.1:1786172102.799433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e0921f0c0. 00010000:00000010:0.1:1786172102.895736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d82400. 00010000:00000010:0.1:1786172103.658813:0:177750:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e093b8000. 00010000:00000010:0.1:1786172107.112607:0:177750:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26320c00. 00010000:00000010:0.1:1786172107.841403:0:178005:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321000. 00010000:00000010:0.1:1786172108.077451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26320a00. 00000080:00000010:0.1:1786172109.876798:0:165139:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf9e0. 00010000:00000010:0.1:1786172109.876825:0:165139:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d275e00. 00010000:00000010:0.1:1786172109.876832:0:165139:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e09289200. 00010000:00000010:0.1:1786172109.876837:0:165139:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1025a600. 00010000:00000010:0.1:1786172109.876839:0:165139:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f4b400. 00000080:00000010:0.1:1786172109.901433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776520. 00000080:00000010:0.1:1786172110.084479:0:178190:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f0f380. 00010000:00000010:0.1:1786172110.091580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f9d4e00. 00010000:00000010:0.1:1786172110.091592:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10e41800. 00010000:00000010:0.1:1786172110.091596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32aff200. 00010000:00000010:0.1:1786172110.091600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318ede00. 00010000:00000010:0.1:1786172110.137389:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d600. 00000080:00000010:0.1:1786172110.386686:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcac20. 00000080:00000010:0.1:1786172110.386708:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119bdac0. 00010000:00000010:0.1:1786172110.389426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e624200. 00010000:00000010:0.1:1786172110.389434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e250c5700. 00010000:00000010:0.1:1786172110.389437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e36a00. 00000080:00000010:0.1:1786172110.392805:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86120. 00000080:00000010:0.1:1786172110.397394:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119ba100. 00010000:00000010:0.1:1786172110.397403:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e625200. 00010000:00000010:0.1:1786172110.397408:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e491d00. 00010000:00000010:0.1:1786172110.397411:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39fbbe00. 00000080:00000010:0.1:1786172110.403405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce8d20. 00000080:00000010:0.1:1786172110.407449:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11955ac0. 00010000:00000010:0.1:1786172110.407468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12f06000. 00010000:00000010:0.1:1786172110.407475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e2b5f00. 00010000:00000010:0.1:1786172110.407480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b57400. 00000080:00000010:0.1:1786172110.416525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce8680. 00000080:00000010:0.1:1786172110.445132:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11a6e300. 00000080:00000010:0.1:1786172110.445142:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca180. 00010000:00000010:0.1:1786172110.793783:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11c9fe00. 00010000:00000010:0.1:1786172110.793798:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2637b800. 00010000:00000010:0.1:1786172110.932384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d800. 00000080:00000010:0.1:1786172111.014231:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e261362e0. 00000080:00000010:0.1:1786172111.046851:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09436b40. 00010000:00000010:0.1:1786172111.046882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:0.1:1786172111.046886:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e32bed900. 00010000:00000010:0.1:1786172111.046889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:0.1:1786172111.046891:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39420800. 00000080:00000010:0.1:1786172111.152304:0:13:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15cb1080. 00010000:00000010:0.1:1786172111.152320:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d0ace00. 00010000:00000010:0.1:1786172111.152324:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d82f800. 00010000:00000010:0.1:1786172111.152329:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08c43700. 00010000:00000010:0.1:1786172111.152333:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12c9ae00. 00010000:00000010:0.1:1786172111.473414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2f600. 00010000:00000010:0.1:1786172111.473446:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31ec0e00. 00010000:00000010:0.1:1786172111.473450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e06091800. 00000080:00000010:0.1:1786172111.505526:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1d40. 00010000:00000010:0.1:1786172111.678561:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31293200. 00010000:00000010:0.1:1786172111.678574:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2c000. 00010000:00000010:0.1:1786172111.678579:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e498400. 00010000:00000010:0.1:1786172111.678582:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2e600. 00000080:00000010:0.1:1786172111.695412:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f5a100. 00000080:00000010:0.1:1786172111.695447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31d2e3e0. 00000080:00000010:0.1:1786172111.695455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1340. 00010000:00000010:0.1:1786172111.910416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31c8a600. 00010000:00000010:0.1:1786172111.910460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:0.1:1786172111.910466:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0925c600. 00010000:00000010:0.1:1786172111.910471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d0f4400. 00010000:00000010:0.1:1786172111.958676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024e600. 00010000:00000010:0.1:1786172111.958682:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10872e00. 00010000:00000010:0.1:1786172111.958686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e08c40400. 00010000:00000010:0.1:1786172111.958689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:0.1:1786172112.248396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f977600. 00010000:00000010:0.1:1786172112.248411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d200. 00010000:00000010:0.1:1786172112.248417:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e06815a00. 00010000:00000010:0.1:1786172112.248431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e092f4c00. 00000080:00000010:0.1:1786172112.272488:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0946ca40. 00000080:00000010:0.1:1786172112.272498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776580. 00010000:00000010:0.1:1786172112.282779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd400. 00010000:00000010:0.1:1786172112.282788:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11c9f500. 00010000:00000010:0.1:1786172112.282793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e720800. 00010000:00000010:0.1:1786172112.282797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31228a00. 00000080:00000010:0.1:1786172112.290550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c866c0. 00000080:00000010:0.1:1786172112.290583:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09b1eb40. 00010000:00000010:0.1:1786172112.290595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31228600. 00010000:00000010:0.1:1786172112.290601:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e03044500. 00010000:00000010:0.1:1786172112.290605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31063000. 00000080:00000010:0.1:1786172112.302560:0:166014:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c864e0. 00000080:00000010:0.1:1786172112.302717:0:166014:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f75ac0. 00010000:00000010:0.1:1786172112.302726:0:166014:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31228000. 00010000:00000010:0.1:1786172112.302731:0:166014:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11c9f000. 00010000:00000010:0.1:1786172112.302736:0:166014:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e720c00. 00000080:00000010:0.1:1786172112.313416:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcab40. 00000080:00000010:0.1:1786172112.313449:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09b1c200. 00000080:00000010:0.1:1786172112.313458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e261366e0. 00010000:00000010:0.1:1786172112.320411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321800. 00000080:00000010:0.1:1786172112.424559:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f6dac0. 00010000:00000010:0.1:1786172112.424589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32904c00. 00010000:00000010:0.1:1786172112.424613:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e07a9f100. 00010000:00000010:0.1:1786172112.424618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32162a00. 00000080:00000010:0.1:1786172112.432408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e26136160. 00000080:00000010:0.1:1786172112.435537:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f6b9c0. 00010000:00000010:0.1:1786172112.435596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00010000:00000010:0.1:1786172112.435604:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0929fc00. 00010000:00000010:0.1:1786172112.435608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180d600. 00010000:00000010:0.1:1786172112.435611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32905600. 00000080:00000010:0.1:1786172112.447431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce8fc0. 00000080:00000010:0.1:1786172112.632443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed16c0. 00000080:00000010:0.1:1786172112.632478:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15ce18c0. 00000080:00000010:0.1:1786172112.632484:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776a80. 00000080:00000010:0.1:1786172112.674402:0:2386:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13f70840. 00010000:00000010:0.1:1786172112.674414:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e62b400. 00010000:00000010:0.1:1786172112.674431:0:2386:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39c0eb00. 00010000:00000010:0.1:1786172112.674435:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32163400. 00010000:00000010:0.1:1786172113.051749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321200. 00010000:00000010:0.1:1786172113.079777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31072e00. 00010000:00000010:0.1:1786172113.079799:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10a9c300. 00010000:00000010:0.1:1786172113.079805:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31d89400. 00010000:00000010:0.1:1786172113.079809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31073200. 00000080:00000010:0.1:1786172113.098647:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7080. 00000080:00000010:0.1:1786172113.098689:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a24e300. 00000080:00000010:0.1:1786172113.098696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7ba0. 00010000:00000010:0.1:1786172113.120442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321200. 00010000:00000010:0.1:1786172113.120455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321400. 00010000:00000010:0.1:1786172113.120470:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e3178a500. 00010000:00000010:0.1:1786172113.120475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31072200. 00010000:00000010:0.1:1786172113.120478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f36400. 00000080:00000010:0.1:1786172113.129738:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a248000. 00000080:00000010:0.1:1786172113.129758:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbdc40. 00010000:00000010:0.1:1786172113.133696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f36000. 00010000:00000010:0.1:1786172113.133712:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e32aef000. 00010000:00000010:0.1:1786172113.133738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527ee00. 00000080:00000010:0.1:1786172113.145892:0:2386:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7780. 00000080:00000010:0.1:1786172113.166424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed16e0. 00010000:00000010:0.1:1786172113.540421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321600. 00010000:00000010:0.1:1786172113.540433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:0.1:1786172113.540438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e3ce800. 00010000:00000010:0.1:1786172113.540443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f34e00. 00000080:00000010:0.1:1786172113.545399:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a318000. 00000080:00000010:0.1:1786172113.545427:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776420. 00010000:00000010:0.1:1786172113.545437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e360000. 00010000:00000010:0.1:1786172113.545442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1513f900. 00010000:00000010:0.1:1786172113.545445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d223800. 00000080:00000010:0.1:1786172113.549397:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776660. 00000080:00000010:0.1:1786172113.553419:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119998c0. 00010000:00000010:0.1:1786172113.564919:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321800. 00010000:00000010:0.1:1786172113.564932:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0f973d00. 00010000:00000010:0.1:1786172113.564939:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2f000. 00010000:00000010:0.1:1786172113.564944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2c800. 00000080:00000010:0.1:1786172113.577665:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd080. 00000080:00000010:0.1:1786172113.583443:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15cb3180. 00010000:00000010:0.1:1786172113.592687:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321800. 00010000:00000010:0.1:1786172113.600396:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e316e9500. 00010000:00000010:0.1:1786172113.600403:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b2cc00. 00000080:00000010:0.1:1786172113.608623:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd540. 00000080:00000010:0.1:1786172113.608651:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f59080. 00010000:00000010:0.1:1786172113.611610:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321800. 00010000:00000010:0.1:1786172113.611617:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10d84500. 00010000:00000010:0.1:1786172113.611623:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31294000. 00000080:00000010:0.1:1786172113.621551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd080. 00000080:00000010:0.1:1786172113.915400:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119bc200. 00000080:00000010:0.1:1786172113.915423:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776280. 00010000:00000010:0.1:1786172113.922439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39fbb000. 00010000:00000010:0.1:1786172113.922448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10981900. 00010000:00000010:0.1:1786172113.922470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e08c14c00. 00010000:00000010:0.1:1786172113.922473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e62dc00. 00000080:00000010:0.1:1786172113.926397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86b60. 00000080:00000010:0.1:1786172113.926433:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1191eb40. 00010000:00000010:0.1:1786172113.926443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e36c00. 00010000:00000010:0.1:1786172113.926449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e250c5000. 00010000:00000010:0.1:1786172113.926453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e624a00. 00000080:00000010:0.1:1786172113.944223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c865c0. 00000080:00000010:0.1:1786172113.952432:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c8a100. 00000080:00000010:0.1:1786172113.952444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca120. 00010000:00000010:0.1:1786172113.957390:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e329ca800. 00010000:00000010:0.1:1786172113.957396:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26320600. 00010000:00000010:0.1:1786172113.957400:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10981b00. 00010000:00000010:0.1:1786172113.957403:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e62c600. 00000080:00000010:0.1:1786172113.965794:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11918840. 00000080:00000010:0.1:1786172113.965817:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd640. 00000080:00000010:0.1:1786172113.965825:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcac80. 00010000:00000010:0.1:1786172113.979529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12f07200. 00010000:00000010:0.1:1786172113.979542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26320600. 00010000:00000010:0.1:1786172113.979553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e2b5900. 00010000:00000010:0.1:1786172113.979558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e62de00. 00010000:00000010:0.1:1786172114.232535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d800. 00010000:00000010:0.1:1786172114.320412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2da00. 00010000:00000010:0.1:1786172114.335473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09248200. 00010000:00000010:0.1:1786172114.335488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:0.1:1786172114.335493:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e124d9f00. 00010000:00000010:0.1:1786172114.335499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da1bc00. 00000080:00000010:0.1:1786172114.357934:0:178192:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09a2eb40. 00000080:00000010:0.1:1786172114.357946:0:178192:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776de0. 00000080:00000010:0.1:1786172114.434295:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11a6a940. 00000080:00000010:0.1:1786172114.434307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86280. 00000080:00000010:0.1:1786172114.463511:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09b1f380. 00000080:00000010:0.1:1786172114.463532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbdbc0. 00010000:00000010:0.1:1786172114.469512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1058ca00. 00010000:00000010:0.1:1786172114.469531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0f7b00. 00010000:00000010:0.1:1786172114.469535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e252a9000. 00000080:00000010:0.1:1786172114.483422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e26136060. 00000080:00000010:0.1:1786172114.492429:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f0b9c0. 00010000:00000010:0.1:1786172114.492448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180e200. 00010000:00000010:0.1:1786172114.492453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1da33600. 00010000:00000010:0.1:1786172114.492457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e6f8600. 00000080:00000010:0.1:1786172114.500504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e055b5e20. 00010000:00000010:0.1:1786172114.648709:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e06155000. 00010000:00000010:0.1:1786172114.648716:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:0.1:1786172114.648719:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39c0e000. 00010000:00000010:0.1:1786172114.648723:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e62b000. 00000080:00000010:0.1:1786172114.660207:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13f74200. 00000080:00000010:0.1:1786172114.660217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776640. 00010000:00000010:0.1:1786172114.660258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da1a400. 00010000:00000010:0.1:1786172114.660263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11cd8100. 00010000:00000010:0.1:1786172114.660268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e313bea00. 00010000:00000010:0.1:1786172114.660272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e06155a00. 00000080:00000010:0.1:1786172114.665419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5d40. 00000080:00000010:0.1:1786172114.665448:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d75280. 00010000:00000010:0.1:1786172114.665455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:0.1:1786172114.665460:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08763400. 00010000:00000010:0.1:1786172114.665465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da1a200. 00000080:00000010:0.1:1786172114.672557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e04efe680. 00000080:00000010:0.1:1786172114.672575:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09a28840. 00010000:00000010:0.1:1786172114.690239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f9d5200. 00010000:00000010:0.1:1786172114.690255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da1b800. 00010000:00000010:0.1:1786172114.690260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e250c4400. 00010000:00000010:0.1:1786172114.690264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942800. 00000080:00000010:0.1:1786172114.703708:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d74a40. 00000080:00000010:0.1:1786172114.703722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb59e0. 00010000:00000010:0.1:1786172114.703764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e554a00. 00010000:00000010:0.1:1786172114.703783:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11cd8f00. 00010000:00000010:0.1:1786172114.703787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d8f8200. 00010000:00000010:0.1:1786172114.703790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943800. 00000080:00000010:0.1:1786172114.703794:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e04efe980. 00000080:00000010:0.1:1786172114.703804:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09a2e300. 00010000:00000010:0.1:1786172114.706673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e554000. 00010000:00000010:0.1:1786172114.706689:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02f25400. 00010000:00000010:0.1:1786172114.706693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e554400. 00000080:00000010:0.1:1786172114.710658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5340. 00000080:00000010:0.1:1786172114.710686:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02360000. 00010000:00000010:0.1:1786172114.710692:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e554800. 00010000:00000010:0.1:1786172114.710697:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11f8f300. 00010000:00000010:0.1:1786172114.710701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e06154400. 00000080:00000010:0.1:1786172114.729458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce88a0. 00010000:00000010:0.1:1786172114.787198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f9d5400. 00010000:00000010:0.1:1786172114.787205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786172114.787209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11f8f000. 00010000:00000010:0.1:1786172114.787213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da1a800. 00000080:00000010:0.1:1786172114.792441:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09a2b180. 00000080:00000010:0.1:1786172114.792460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1840. 00000080:00000010:0.1:1786172114.795654:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1193e300. 00010000:00000010:0.1:1786172114.795677:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318f4600. 00010000:00000010:0.1:1786172114.795682:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d260200. 00010000:00000010:0.1:1786172115.095724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d600. 00010000:00000010:0.1:1786172115.160408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abc00. 00010000:00000010:0.1:1786172115.160418:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08763e00. 00010000:00000010:0.1:1786172115.160423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da1b000. 00010000:00000010:0.1:1786172115.206071:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da1b400. 00010000:00000010:0.1:1786172115.206084:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08763900. 00010000:00000010:0.1:1786172115.206089:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f9d5800. 00010000:00000010:0.1:1786172115.206093:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321000. 00010000:00000010:0.1:1786172115.206095:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08763100. 00010000:00000010:0.1:1786172115.206115:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da1ba00. 00000080:00000010:0.1:1786172115.210154:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09a2ca40. 00000080:00000010:0.1:1786172115.210175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117767a0. 00010000:00000010:0.1:1786172115.210185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d223000. 00010000:00000010:0.1:1786172115.210190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0907e900. 00010000:00000010:0.1:1786172115.210195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d161c00. 00000080:00000010:0.1:1786172115.217424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbfb80. 00010000:00000010:0.1:1786172115.267491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e624c00. 00010000:00000010:0.1:1786172115.267497:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e03044a00. 00010000:00000010:0.1:1786172115.267500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da1a000. 00000080:00000010:0.1:1786172115.566492:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13f739c0. 00010000:00000010:0.1:1786172115.566504:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180de00. 00010000:00000010:0.1:1786172115.566636:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02f25600. 00010000:00000010:0.1:1786172115.566642:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32904a00. 00000080:00000010:0.1:1786172115.566648:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5260. 00000080:00000010:0.1:1786172115.566658:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09593180. 00000080:00000010:0.1:1786172115.581479:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb57a0. 00000080:00000010:0.1:1786172115.581500:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09590840. 00010000:00000010:0.1:1786172115.581509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26192a00. 00010000:00000010:0.1:1786172115.581515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0929fb00. 00010000:00000010:0.1:1786172115.581520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa600. 00000080:00000010:0.1:1786172115.584657:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5540. 00000080:00000010:0.1:1786172115.584675:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13f72100. 00010000:00000010:0.1:1786172115.584684:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e06154e00. 00010000:00000010:0.1:1786172115.584689:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39c0ee00. 00010000:00000010:0.1:1786172115.584693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e313be000. 00000080:00000010:0.1:1786172115.700616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf620. 00010000:00000010:0.1:1786172115.993500:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e32924d00. 00010000:00000010:0.1:1786172115.993509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024c400. 00010000:00000010:0.1:1786172116.005504:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08c43100. 00010000:00000010:0.1:1786172116.005511:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d167200. 00010000:00000010:0.1:1786172116.005514:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e498600. 00010000:00000010:0.1:1786172116.005533:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d0ad800. 00010000:00000010:0.1:1786172116.191308:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31c4bf00. 00010000:00000010:0.1:1786172116.191317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02fa5800. 00010000:00000010:0.1:1786172116.191322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31c4bc00. 00010000:00000010:0.1:1786172116.191325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180da00. 00010000:00000010:0.1:1786172116.191332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e32bed800. 00010000:00000010:0.1:1786172116.191336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e092e8000. 00010000:00000010:0.1:1786172116.191340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0929f300. 00010000:00000010:0.1:1786172116.191343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e315dd200. 00010000:00000010:0.1:1786172116.208433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11f8f800. 00010000:00000010:0.1:1786172116.208445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32163600. 00010000:00000010:0.1:1786172116.215418:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39c0e700. 00010000:00000010:0.1:1786172116.215429:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02fa5000. 00010000:00000010:0.1:1786172116.215434:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0929f500. 00010000:00000010:0.1:1786172116.215437:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:0.1:1786172116.215441:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e07cee800. 00010000:00000010:0.1:1786172116.215444:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39fbb400. 00010000:00000010:0.1:1786172116.277626:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:0.1:1786172116.277633:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31063c00. 00010000:00000010:0.1:1786172116.294208:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e03044c00. 00010000:00000010:0.1:1786172116.294217:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0576a400. 00010000:00000010:0.1:1786172116.294221:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e07cee900. 00010000:00000010:0.1:1786172116.294226:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e08c15400. 00010000:00000010:0.1:1786172116.431419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180a400. 00010000:00000010:0.1:1786172116.431435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d800. 00010000:00000010:0.1:1786172116.431441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0faabf00. 00010000:00000010:0.1:1786172116.431445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f924e00. 00000080:00000010:0.1:1786172116.447415:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32fe6300. 00000080:00000010:0.1:1786172116.447438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117768c0. 00010000:00000010:0.1:1786172116.447471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e055e9e00. 00010000:00000010:0.1:1786172116.447477:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8b00. 00010000:00000010:0.1:1786172116.447481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e055e8400. 00010000:00000010:0.1:1786172116.447501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180b000. 00000080:00000010:0.1:1786172116.455446:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092962e0. 00000080:00000010:0.1:1786172116.455463:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32fe2940. 00000080:00000010:0.1:1786172116.455468:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296200. 00010000:00000010:0.1:1786172116.468499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f924000. 00010000:00000010:0.1:1786172116.468505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321600. 00010000:00000010:0.1:1786172116.468510:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8e00. 00010000:00000010:0.1:1786172116.468513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e055e8200. 00000080:00000010:0.1:1786172116.473875:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32fe7380. 00000080:00000010:0.1:1786172116.473892:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbdac0. 00010000:00000010:0.1:1786172116.499454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39421200. 00010000:00000010:0.1:1786172116.499460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1025d200. 00010000:00000010:0.1:1786172116.499465:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3d00. 00010000:00000010:0.1:1786172116.499469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1025d800. 00010000:00000010:0.1:1786172116.499472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26321400. 00000080:00000010:0.1:1786172116.524507:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32fe39c0. 00000080:00000010:0.1:1786172116.524572:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296fc0. 00010000:00000010:0.1:1786172116.524587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e055e8600. 00010000:00000010:0.1:1786172116.524593:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8900. 00000080:00000010:0.1:1786172116.652483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e26136460. 00000080:00000010:0.1:1786172116.659453:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15df8000. 00010000:00000010:0.1:1786172116.667403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e6fec00. 00010000:00000010:0.1:1786172116.667413:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10e41f00. 00010000:00000010:0.1:1786172116.667417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32afee00. 00010000:00000010:0.1:1786172116.667420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e6ff800. 00000080:00000010:0.1:1786172116.677407:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca2e0. 00000080:00000010:0.1:1786172116.677427:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09405ac0. 00010000:00000010:0.1:1786172116.884416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10ba5e00. 00010000:00000010:0.1:1786172116.884434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08763b00. 00010000:00000010:0.1:1786172116.884438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217ce00. 00000080:00000010:0.1:1786172116.896905:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5740. 00000080:00000010:0.1:1786172116.896935:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13e46b40. 00010000:00000010:0.1:1786172116.896942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32aff400. 00010000:00000010:0.1:1786172116.896969:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e09308300. 00010000:00000010:0.1:1786172116.896974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32185e00. 00000080:00000010:0.1:1786172116.905421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce8000. 00000080:00000010:0.1:1786172116.905451:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32c498c0. 00010000:00000010:0.1:1786172116.905461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09248e00. 00010000:00000010:0.1:1786172116.905466:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0faad300. 00010000:00000010:0.1:1786172116.905469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f926800. 00000080:00000010:0.1:1786172116.919426:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c865e0. 00000080:00000010:0.1:1786172116.919454:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13e40840. 00010000:00000010:0.1:1786172116.919463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f927600. 00010000:00000010:0.1:1786172116.919469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0faad500. 00010000:00000010:0.1:1786172116.919473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f926200. 00000080:00000010:0.1:1786172116.943913:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86800. 00000080:00000010:0.1:1786172116.943935:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13e46300. 00010000:00000010:0.1:1786172116.943944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f927200. 00010000:00000010:0.1:1786172116.943949:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e2b5500. 00010000:00000010:0.1:1786172116.943954:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25170800. 00000080:00000010:0.1:1786172116.943956:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86fa0. 00000080:00000010:0.1:1786172116.943961:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11a6ca40. 00010000:00000010:0.1:1786172116.949678:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31229e00. 00010000:00000010:0.1:1786172116.949696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318edc00. 00010000:00000010:0.1:1786172116.949702:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e09308900. 00010000:00000010:0.1:1786172116.949722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e6fe000. 00000080:00000010:0.1:1786172116.963894:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32df1080. 00000080:00000010:0.1:1786172116.963913:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce8d80. 00000080:00000010:0.1:1786172116.963921:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce8e40. 00010000:00000010:0.1:1786172116.973420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31228800. 00010000:00000010:0.1:1786172116.973437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26320400. 00010000:00000010:0.1:1786172116.973443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10e41d00. 00010000:00000010:0.1:1786172116.973448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32affe00. 00000080:00000010:0.1:1786172116.989613:0:2385:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32c4a100. 00000080:00000010:0.1:1786172116.989623:0:2385:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd160. 00000080:00000010:0.1:1786172117.180328:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1193c200. 00000080:00000010:0.1:1786172117.180386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf700. 00010000:00000010:0.1:1786172117.180416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f9e600. 00010000:00000010:0.1:1786172117.180422:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10b39600. 00010000:00000010:0.1:1786172117.180427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12f09000. 00000080:00000010:0.1:1786172117.365417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86ca0. 00000080:00000010:0.1:1786172117.365438:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09b40840. 00000080:00000010:0.1:1786172117.365445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf040. 00010000:00000010:0.1:1786172118.035590:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e32aefd00. 00010000:00000010:0.1:1786172118.035606:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527f800. 00010000:00000010:0.1:1786172118.035609:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e32aefa00. 00010000:00000010:0.1:1786172118.035613:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09310000. 00010000:00000010:0.1:1786172118.035616:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1da33200. 00010000:00000010:0.1:1786172118.035620:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12f09400. 00010000:00000010:0.1:1786172118.080732:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0f7000. 00010000:00000010:0.1:1786172118.080747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26278c00. 00010000:00000010:0.1:1786172118.080752:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e51cb00. 00010000:00000010:0.1:1786172118.080755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d160e00. 00010000:00000010:0.1:1786172118.098755:0:2386:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebb00. 00010000:00000010:0.1:1786172118.098764:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31228200. 00010000:00000010:0.1:1786172118.115413:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11cd8e00. 00010000:00000010:0.1:1786172118.115423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d8f8600. 00010000:00000010:0.1:1786172118.120420:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02f25b00. 00010000:00000010:0.1:1786172118.120439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943400. 00010000:00000010:0.1:1786172118.140400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0907e200. 00010000:00000010:0.1:1786172118.140410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31611000. 00010000:00000010:0.1:1786172118.140414:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31b10d00. 00010000:00000010:0.1:1786172118.140418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12f08c00. 00010000:00000010:0.1:1786172118.140425:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418400. 00010000:00000010:0.1:1786172118.140432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31292c00. 00010000:00000010:0.1:1786172118.140439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0faacb00. 00010000:00000010:0.1:1786172118.140445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3293c200. 00010000:00000010:0.1:1786172118.140454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1da1d800. 00010000:00000010:0.1:1786172118.140478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2637a000. 00010000:00000010:0.1:1786172118.196741:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0907ee00. 00010000:00000010:0.1:1786172118.196750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e361800. 00010000:00000010:0.1:1786172118.207400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0568a400. 00010000:00000010:0.1:1786172118.207417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318e9800. 00010000:00000010:0.1:1786172118.207420:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e09289000. 00010000:00000010:0.1:1786172118.207423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e092fae00. 00010000:00000010:0.1:1786172118.280451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e6f9a00. 00010000:00000010:0.1:1786172118.280465:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10db0200. 00010000:00000010:0.1:1786172118.280470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e263a6a00. 00010000:00000010:0.1:1786172118.280473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e6f8400. 00000080:00000010:0.1:1786172118.495400:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1b40. 00000080:00000010:0.1:1786172118.495835:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099d1080. 00010000:00000010:0.1:1786172118.495845:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31a04200. 00010000:00000010:0.1:1786172118.495850:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e318ea900. 00010000:00000010:0.1:1786172118.495854:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11f8d800. 00000080:00000010:0.1:1786172118.495857:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1f00. 00000080:00000010:0.1:1786172118.499880:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09976b40. 00010000:00000010:0.1:1786172118.499893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d82c200. 00010000:00000010:0.1:1786172118.499897:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e12e66400. 00010000:00000010:0.1:1786172118.499899:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31a22600. 00000080:00000010:0.1:1786172118.510638:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1a40. 00000080:00000010:0.1:1786172118.510669:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119818c0. 00010000:00000010:0.1:1786172118.510680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d82d600. 00010000:00000010:0.1:1786172118.510687:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230900. 00010000:00000010:0.1:1786172118.510691:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0939f400. 00000080:00000010:0.1:1786172118.570695:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1100. 00000080:00000010:0.1:1786172118.570719:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09976300. 00010000:00000010:0.1:1786172118.570724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d82dc00. 00010000:00000010:0.1:1786172118.570727:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e318ea800. 00010000:00000010:0.1:1786172118.570730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11f8dc00. 00010000:00000010:0.1:1786172118.621558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312f4200. 00010000:00000010:0.1:1786172118.621590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e15b57b00. 00010000:00000010:0.1:1786172118.808617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1025c600. 00010000:00000010:0.1:1786172118.808626:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3f00. 00010000:00000010:0.1:1786172118.808631:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1025de00. 00000080:00000010:0.1:1786172118.819676:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296920. 00000080:00000010:0.1:1786172118.819701:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d70840. 00010000:00000010:0.1:1786172118.819713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1025da00. 00010000:00000010:0.1:1786172118.819719:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8200. 00010000:00000010:0.1:1786172118.819723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1025d400. 00000080:00000010:0.1:1786172118.887481:0:2385:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094d18c0. 00010000:00000010:0.1:1786172118.887491:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e315dd400. 00010000:00000010:0.1:1786172118.887496:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31c4b400. 00010000:00000010:0.1:1786172118.887502:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31109600. 00010000:00000010:0.1:1786172118.900411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31e8a600. 00010000:00000010:0.1:1786172118.900428:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3000. 00010000:00000010:0.1:1786172118.904534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10dea000. 00000080:00000010:0.1:1786172119.074414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce8420. 00000080:00000010:0.1:1786172119.074441:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d46b40. 00010000:00000010:0.1:1786172119.078016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d1e9c00. 00010000:00000010:0.1:1786172119.078023:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9400. 00010000:00000010:0.1:1786172119.078027:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31e8a400. 00000080:00000010:0.1:1786172119.089659:0:2386:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092968c0. 00000080:00000010:0.1:1786172119.089682:0:2386:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d40000. 00010000:00000010:0.1:1786172119.089689:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31108800. 00010000:00000010:0.1:1786172119.089695:0:2386:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e32ad7700. 00010000:00000010:0.1:1786172119.089698:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d1e9400. 00000080:00000010:0.1:1786172119.139408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092966e0. 00000080:00000010:0.1:1786172119.139430:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d20000. 00010000:00000010:0.1:1786172119.139437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f925c00. 00010000:00000010:0.1:1786172119.139441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3e00. 00010000:00000010:0.1:1786172119.139445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10dea200. 00010000:00000010:0.1:1786172119.160492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e315dda00. 00010000:00000010:0.1:1786172119.160571:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3600. 00010000:00000010:0.1:1786172119.160577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d1e9200. 00010000:00000010:0.1:1786172119.264426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12c65c00. 00010000:00000010:0.1:1786172119.264442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786172119.264447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230f00. 00010000:00000010:0.1:1786172119.264451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e06090400. 00000080:00000010:0.1:1786172119.298127:0:2386:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0984e300. 00000080:00000010:0.1:1786172119.298142:0:2386:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1920. 00010000:00000010:0.1:1786172119.668398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26279a00. 00010000:00000010:0.1:1786172119.668413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26320c00. 00010000:00000010:0.1:1786172119.668417:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e139fe500. 00010000:00000010:0.1:1786172119.668422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e329cb600. 00000080:00000010:0.1:1786172119.792466:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02361080. 00010000:00000010:0.1:1786172119.792484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:0.1:1786172119.792491:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e626c00. 00010000:00000010:0.1:1786172119.792496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e060f0200. 00000080:00000010:0.1:1786172119.809416:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce8960. 00000080:00000010:0.1:1786172119.809446:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02362940. 00010000:00000010:0.1:1786172119.809455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02fa4c00. 00010000:00000010:0.1:1786172119.809471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08763c00. 00010000:00000010:0.1:1786172119.809476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e060f1000. 00000080:00000010:0.1:1786172119.817258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce8ac0. 00000080:00000010:0.1:1786172119.817289:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d77380. 00010000:00000010:0.1:1786172119.836709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc800. 00010000:00000010:0.1:1786172119.836725:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11f8ff00. 00010000:00000010:0.1:1786172119.836729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00000080:00000010:0.1:1786172119.903087:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce8da0. 00000080:00000010:0.1:1786172119.903117:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32c48000. 00000080:00000010:0.1:1786172119.903126:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e04efe5a0. 00010000:00000010:0.1:1786172120.018431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f969600. 00010000:00000010:0.1:1786172120.018442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e626e00. 00010000:00000010:0.1:1786172120.018446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f969200. 00010000:00000010:0.1:1786172120.018466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09248c00. 00010000:00000010:0.1:1786172120.018469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f968200. 00000080:00000010:0.1:1786172120.031437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5da0. 00000080:00000010:0.1:1786172120.031469:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32c4eb40. 00010000:00000010:0.1:1786172120.043439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32affa00. 00010000:00000010:0.1:1786172120.043463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:0.1:1786172120.043468:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10de5000. 00010000:00000010:0.1:1786172120.043473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00000080:00000010:0.1:1786172120.067417:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02362100. 00000080:00000010:0.1:1786172120.067446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e04efe0c0. 00010000:00000010:0.1:1786172120.067458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786172120.067464:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02e31400. 00010000:00000010:0.1:1786172120.067468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8c00. 00000080:00000010:0.1:1786172120.083659:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e04efe4c0. 00000080:00000010:0.1:1786172120.083688:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02365ac0. 00010000:00000010:0.1:1786172120.093403:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:0.1:1786172120.093411:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0faad000. 00010000:00000010:0.1:1786172120.093416:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8200. 00000080:00000010:0.1:1786172120.108416:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5720. 00000080:00000010:0.1:1786172120.108461:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e023639c0. 00010000:00000010:0.1:1786172120.113562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26192200. 00010000:00000010:0.1:1786172120.113571:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e32bed500. 00010000:00000010:0.1:1786172120.113575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00000080:00000010:0.1:1786172120.118517:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5a20. 00010000:00000010:0.1:1786172120.156556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10ba4200. 00010000:00000010:0.1:1786172120.156573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09249000. 00010000:00000010:0.1:1786172120.156579:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e626a00. 00010000:00000010:0.1:1786172120.156583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8e00. 00000080:00000010:0.1:1786172120.165047:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02367380. 00000080:00000010:0.1:1786172120.165064:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5b00. 00010000:00000010:0.1:1786172120.169488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31a0e400. 00010000:00000010:0.1:1786172120.169499:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08763600. 00010000:00000010:0.1:1786172120.169504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f968000. 00010000:00000010:0.1:1786172120.169521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09248a00. 00000080:00000010:0.1:1786172120.174452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5d00. 00000080:00000010:0.1:1786172120.174484:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13e44a40. 00000080:00000010:0.1:1786172120.370787:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5780. 00000080:00000010:0.1:1786172120.375406:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32c4ca40. 00010000:00000010:0.1:1786172120.375415:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11f07c00. 00010000:00000010:0.1:1786172120.375419:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0faadb00. 00010000:00000010:0.1:1786172120.375422:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f926000. 00000080:00000010:0.1:1786172120.380405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5620. 00000080:00000010:0.1:1786172120.388400:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32df4a40. 00010000:00000010:0.1:1786172120.463406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d400. 00010000:00000010:0.1:1786172120.577419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d800. 00010000:00000010:0.1:1786172120.886554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720ef00. 00010000:00000010:0.1:1786172120.886568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b56400. 00010000:00000010:0.1:1786172120.886572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818fe00. 00010000:00000010:0.1:1786172120.886575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e314e3600. 00010000:00000010:0.1:1786172120.886579:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f300. 00010000:00000010:0.1:1786172120.886581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e314e3e00. 00010000:00000010:0.1:1786172120.891409:0:152386:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e029b3400. 00010000:00000010:0.1:1786172120.891417:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f40200. 00010000:00000010:0.1:1786172120.934412:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02f25900. 00010000:00000010:0.1:1786172120.934433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31a0ec00. 00000080:00000010:0.1:1786172121.169881:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13e76b40. 00000080:00000010:0.1:1786172121.169903:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776640. 00010000:00000010:0.1:1786172121.176716:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e08c0b000. 00010000:00000010:0.1:1786172121.176728:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31a02100. 00010000:00000010:0.1:1786172121.176732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11f05e00. 00000080:00000010:0.1:1786172121.185425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf420. 00000080:00000010:0.1:1786172121.185470:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c18840. 00010000:00000010:0.1:1786172121.189708:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e6f8000. 00010000:00000010:0.1:1786172121.189714:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372a00. 00010000:00000010:0.1:1786172121.189718:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e08032900. 00010000:00000010:0.1:1786172121.189732:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e88600. 00000080:00000010:0.1:1786172121.196413:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13e74200. 00000080:00000010:0.1:1786172121.196425:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86ee0. 00010000:00000010:0.1:1786172121.196433:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e08c0aa00. 00010000:00000010:0.1:1786172121.196438:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d260d00. 00010000:00000010:0.1:1786172121.196442:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1025ae00. 00000080:00000010:0.1:1786172121.206403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbffa0. 00000080:00000010:0.1:1786172121.209609:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1199eb40. 00010000:00000010:0.1:1786172121.209651:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31c33a00. 00010000:00000010:0.1:1786172121.209655:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e25b91900. 00010000:00000010:0.1:1786172121.209657:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11c85c00. 00010000:00000010:0.1:1786172121.209659:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31c32a00. 00000080:00000010:0.1:1786172121.214422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07c901c0. 00000080:00000010:0.1:1786172121.218418:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0214a940. 00010000:00000010:0.1:1786172121.218436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312a8000. 00010000:00000010:0.1:1786172121.218441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e139fe800. 00010000:00000010:0.1:1786172121.218446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e329cb800. 00000080:00000010:0.1:1786172121.233412:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86400. 00000080:00000010:0.1:1786172121.233430:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119539c0. 00010000:00000010:0.1:1786172121.233440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312a8600. 00010000:00000010:0.1:1786172121.233445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5b00. 00010000:00000010:0.1:1786172121.233448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39412400. 00010000:00000010:0.1:1786172121.260397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:0.1:1786172121.589729:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11075e00. 00010000:00000010:0.1:1786172121.589755:0:2393:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d260700. 00000080:00000010:0.1:1786172121.776435:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099739c0. 00000080:00000010:0.1:1786172121.776455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e258c6ac0. 00010000:00000010:0.1:1786172121.790647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e06092800. 00010000:00000010:0.1:1786172121.790659:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11f8c400. 00010000:00000010:0.1:1786172121.790664:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e15b57100. 00010000:00000010:0.1:1786172121.790668:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31a23000. 00000080:00000010:0.1:1786172121.796729:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099718c0. 00000080:00000010:0.1:1786172121.796772:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e258c6b40. 00010000:00000010:0.1:1786172121.796813:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e06090c00. 00010000:00000010:0.1:1786172121.796818:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e05223400. 00010000:00000010:0.1:1786172121.796823:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0938b200. 00010000:00000010:0.1:1786172121.796827:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31a22a00. 00000080:00000010:0.1:1786172121.801532:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e118f4a40. 00000080:00000010:0.1:1786172121.801549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed14a0. 00000080:00000010:0.1:1786172121.801557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e10b58e40. 00010000:00000010:0.1:1786172121.839452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09312600. 00010000:00000010:0.1:1786172121.839466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:0.1:1786172121.839479:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e15b57a00. 00010000:00000010:0.1:1786172121.839484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e17ae00. 00010000:00000010:0.1:1786172121.839487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e39a00. 00000080:00000010:0.1:1786172121.866415:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e118f1080. 00000080:00000010:0.1:1786172121.866434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd260. 00000080:00000010:0.1:1786172121.915522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e258c6bc0. 00000080:00000010:0.1:1786172121.924088:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e118f6b40. 00000080:00000010:0.1:1786172122.110382:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e10b58c80. 00000080:00000010:0.1:1786172122.114781:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11985ac0. 00010000:00000010:0.1:1786172122.118016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024d000. 00010000:00000010:0.1:1786172122.118028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31a22c00. 00010000:00000010:0.1:1786172122.118033:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31e1be00. 00010000:00000010:0.1:1786172122.118037:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e05222e00. 00000080:00000010:0.1:1786172122.128477:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09972940. 00000080:00000010:0.1:1786172122.128501:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e10b58720. 00010000:00000010:0.1:1786172122.362781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318f5200. 00010000:00000010:0.1:1786172122.362789:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1513f100. 00010000:00000010:0.1:1786172122.362794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e08c0b800. 00000080:00000010:0.1:1786172122.375855:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbffc0. 00000080:00000010:0.1:1786172122.383409:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11980840. 00010000:00000010:0.1:1786172122.383416:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2507f200. 00010000:00000010:0.1:1786172122.383421:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31ec3c00. 00010000:00000010:0.1:1786172122.383424:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d160400. 00000080:00000010:0.1:1786172122.398005:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e10b58640. 00000080:00000010:0.1:1786172122.398034:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09430000. 00000080:00000010:0.1:1786172122.494830:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13e71080. 00010000:00000010:0.1:1786172122.494847:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e251c2a00. 00010000:00000010:0.1:1786172122.494852:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e086dd900. 00010000:00000010:0.1:1786172122.494856:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fac2a00. 00000080:00000010:0.1:1786172122.504660:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12d22ae0. 00000080:00000010:0.1:1786172122.504692:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c8a940. 00010000:00000010:0.1:1786172122.579674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3e00. 00010000:00000010:0.1:1786172122.579691:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac000. 00010000:00000010:0.1:1786172122.579699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e313b9a00. 00000080:00000010:0.1:1786172122.822021:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12d22220. 00000080:00000010:0.1:1786172122.822046:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15ebf380. 00010000:00000010:0.1:1786172122.822054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e08d49800. 00010000:00000010:0.1:1786172122.822060:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a685d00. 00010000:00000010:0.1:1786172122.822064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00000080:00000010:0.1:1786172122.826619:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12d22340. 00000080:00000010:0.1:1786172122.830501:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15eb98c0. 00010000:00000010:0.1:1786172123.033637:0:166014:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31bdfb00. 00010000:00000010:0.1:1786172123.033649:0:166014:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1058d800. 00010000:00000010:0.1:1786172123.042422:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2800. 00010000:00000010:0.1:1786172123.042429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e08d49a00. 00010000:00000010:0.1:1786172123.042432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2d00. 00010000:00000010:0.1:1786172123.042435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e252a9c00. 00010000:00000010:0.1:1786172123.056590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e72f500. 00010000:00000010:0.1:1786172123.056603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e251c2600. 00010000:00000010:0.1:1786172123.056607:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10db0900. 00010000:00000010:0.1:1786172123.056611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e6f9000. 00010000:00000010:0.1:1786172123.056617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31b10f00. 00010000:00000010:0.1:1786172123.056621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e263a6c00. 00010000:00000010:0.1:1786172123.056640:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e029f8400. 00010000:00000010:0.1:1786172123.056644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342400. 00010000:00000010:0.1:1786172123.382179:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10debc00. 00010000:00000010:0.1:1786172123.382203:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180c000. 00010000:00000010:0.1:1786172123.382208:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0929f400. 00010000:00000010:0.1:1786172123.382212:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180c800. 00010000:00000010:0.1:1786172123.382215:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180dc00. 00010000:00000010:0.1:1786172123.399475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1da33800. 00010000:00000010:0.1:1786172123.399481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e6f8c00. 00010000:00000010:0.1:1786172123.955159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:0.1:1786172123.968088:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f4b000. 00010000:00000010:0.1:1786172123.968097:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd200. 00010000:00000010:0.1:1786172123.968102:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f4ae00. 00010000:00000010:0.1:1786172123.968104:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26320200. 00010000:00000010:0.1:1786172123.990955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26202400. 00010000:00000010:0.1:1786172124.103690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343c00. 00010000:00000010:0.1:1786172124.103701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12f06a00. 00010000:00000010:0.1:1786172124.103706:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e139fe000. 00010000:00000010:0.1:1786172124.103711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e317fa000. 00010000:00000010:0.1:1786172124.103715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e317fa600. 00000080:00000010:0.1:1786172124.111659:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0214d280. 00000080:00000010:0.1:1786172124.111677:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca9e0. 00010000:00000010:0.1:1786172124.121654:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342e00. 00010000:00000010:0.1:1786172124.121667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e317fb200. 00010000:00000010:0.1:1786172124.121672:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e250c2e00. 00010000:00000010:0.1:1786172124.121676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312a9000. 00000080:00000010:0.1:1786172124.132443:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11a01080. 00000080:00000010:0.1:1786172124.132473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e26136820. 00010000:00000010:0.1:1786172124.137074:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e317fbe00. 00010000:00000010:0.1:1786172124.137082:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e138ab200. 00010000:00000010:0.1:1786172124.137087:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f41e00. 00010000:00000010:0.1:1786172124.137090:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312a9c00. 00000080:00000010:0.1:1786172124.147553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca0c0. 00000080:00000010:0.1:1786172124.147590:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11a04a40. 00000080:00000010:0.1:1786172124.147599:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86060. 00010000:00000010:0.1:1786172124.158191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343e00. 00010000:00000010:0.1:1786172124.158199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786172124.158205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e250c2300. 00010000:00000010:0.1:1786172124.158209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11c85200. 00000080:00000010:0.1:1786172124.164665:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11a05280. 00000080:00000010:0.1:1786172124.164703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd900. 00010000:00000010:0.1:1786172124.171697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342800. 00010000:00000010:0.1:1786172124.171708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e317fbc00. 00010000:00000010:0.1:1786172124.171717:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e250c2500. 00010000:00000010:0.1:1786172124.171725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312a8400. 00000080:00000010:0.1:1786172124.179428:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c8eb40. 00000080:00000010:0.1:1786172124.179446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86c60. 00000080:00000010:0.1:1786172124.188575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12ce8b00. 00010000:00000010:0.1:1786172124.199523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f40800. 00010000:00000010:0.1:1786172124.199535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e25b91500. 00010000:00000010:0.1:1786172124.214658:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343000. 00010000:00000010:0.1:1786172124.214687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:0.1:1786172124.214692:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03000. 00010000:00000010:0.1:1786172124.214697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12f06600. 00010000:00000010:0.1:1786172124.266728:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d83e00. 00010000:00000010:0.1:1786172124.266737:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2da00. 00010000:00000010:0.1:1786172124.266751:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230100. 00010000:00000010:0.1:1786172124.266755:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f858400. 00000080:00000010:0.1:1786172124.280707:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0984eb40. 00000080:00000010:0.1:1786172124.280720:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117764c0. 00010000:00000010:0.1:1786172124.288679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372600. 00010000:00000010:0.1:1786172124.288688:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b56a00. 00010000:00000010:0.1:1786172124.288691:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e491e00. 00010000:00000010:0.1:1786172124.288695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0576b200. 00010000:00000010:0.1:1786172124.288712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12833c00. 00000080:00000010:0.1:1786172124.298427:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11919080. 00000080:00000010:0.1:1786172124.298460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86600. 00010000:00000010:0.1:1786172124.302650:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373800. 00010000:00000010:0.1:1786172124.302669:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39fbb600. 00010000:00000010:0.1:1786172124.302672:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e250c5300. 00010000:00000010:0.1:1786172124.302677:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e37600. 00000080:00000010:0.1:1786172124.311655:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c8dac0. 00000080:00000010:0.1:1786172124.311675:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e12c86900. 00000080:00000010:0.1:1786172124.311683:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e26136660. 00010000:00000010:0.1:1786172124.321784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fac2c00. 00010000:00000010:0.1:1786172124.321792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:0.1:1786172124.321797:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e2b5600. 00010000:00000010:0.1:1786172124.321821:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12832400. 00000080:00000010:0.1:1786172124.331437:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11a6dac0. 00000080:00000010:0.1:1786172124.331467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd800. 00010000:00000010:0.1:1786172124.334729:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e721800. 00010000:00000010:0.1:1786172124.334735:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32184600. 00010000:00000010:0.1:1786172124.334739:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10ba2400. 00010000:00000010:0.1:1786172124.334759:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32185c00. 00010000:00000010:0.1:1786172124.334763:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32185400. 00010000:00000010:0.1:1786172124.334765:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00000080:00000010:0.1:1786172124.345462:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13e47380. 00000080:00000010:0.1:1786172124.345484:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5e20. 00010000:00000010:0.1:1786172124.355608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e721c00. 00010000:00000010:0.1:1786172124.355618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32184000. 00010000:00000010:0.1:1786172124.355622:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02e31300. 00010000:00000010:0.1:1786172124.355626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32affc00. 00010000:00000010:0.1:1786172124.462705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e721200. 00010000:00000010:0.1:1786172124.462734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32184800. 00010000:00000010:0.1:1786172124.462739:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10de5900. 00010000:00000010:0.1:1786172124.462757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11f06200. 00000080:00000010:0.1:1786172124.488656:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32c49080. 00000080:00000010:0.1:1786172124.488674:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e04efe940. 00000080:00000010:0.1:1786172124.518667:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15dfb9c0. 00010000:00000010:0.1:1786172124.537641:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e068c5800. 00010000:00000010:0.1:1786172124.537648:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f41400. 00010000:00000010:0.1:1786172124.537668:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f900. 00010000:00000010:0.1:1786172124.537671:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f41600. 00000080:00000010:0.1:1786172124.546585:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c898c0. 00000080:00000010:0.1:1786172124.546606:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcade0. 00010000:00000010:0.1:1786172124.546621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11f04400. 00010000:00000010:0.1:1786172124.546626:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d237900. 00010000:00000010:0.1:1786172124.550634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f858800. 00010000:00000010:0.1:1786172124.550647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e05225200. 00010000:00000010:0.1:1786172124.550652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d237200. 00010000:00000010:0.1:1786172124.550656:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e05225a00. 00000080:00000010:0.1:1786172124.575676:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c1e300. 00000080:00000010:0.1:1786172124.575688:0:178189:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf220. 00010000:00000010:0.1:1786172124.575695:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318f4200. 00010000:00000010:0.1:1786172124.575699:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1513fd00. 00010000:00000010:0.1:1786172124.575703:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e08c0a600. 00000080:00000010:0.1:1786172124.592408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf6c0. 00000080:00000010:0.1:1786172124.592456:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11980000. 00010000:00000010:0.1:1786172124.592464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e05224200. 00010000:00000010:0.1:1786172124.592469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d237a00. 00010000:00000010:0.1:1786172124.592473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318f4e00. 00000080:00000010:0.1:1786172124.611407:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbff00. 00010000:00000010:0.1:1786172124.732878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318f5600. 00010000:00000010:0.1:1786172124.732896:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31b10b00. 00010000:00000010:0.1:1786172124.732915:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e08c0be00. 00000080:00000010:0.1:1786172124.733645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbff60. 00000080:00000010:0.1:1786172124.738407:0:178189:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1199c200. 00010000:00000010:0.1:1786172124.738853:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e092fa600. 00010000:00000010:0.1:1786172124.738860:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e31b3b800. 00010000:00000010:0.1:1786172124.738864:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e49b400. 00000080:00000010:0.1:1786172124.751506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e10b58980. 00000080:00000010:0.1:1786172124.760471:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15dfdac0. 00000080:00000010:0.1:1786172124.853533:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d718c0. 00010000:00000010:0.1:1786172124.877471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00000080:00000010:0.1:1786172124.913654:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094d6300. 00000080:00000010:0.1:1786172124.913687:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776f40. 00010000:00000010:0.1:1786172124.917620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31611400. 00010000:00000010:0.1:1786172124.917637:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10fced00. 00010000:00000010:0.1:1786172124.917642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d163a00. 00000080:00000010:0.1:1786172124.936320:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e261368e0. 00000080:00000010:0.1:1786172124.936348:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0233b9c0. 00010000:00000010:0.1:1786172124.952982:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31610e00. 00010000:00000010:0.1:1786172124.952998:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0938ae00. 00010000:00000010:0.1:1786172124.953004:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e360c00. 00000080:00000010:0.1:1786172124.953008:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e26136e80. 00000080:00000010:0.1:1786172124.953039:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0233a100. 00010000:00000010:0.1:1786172124.953278:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39fbb800. 00010000:00000010:0.1:1786172124.953282:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e07cee200. 00000080:00000010:0.1:1786172125.116937:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02171080. 00000080:00000010:0.1:1786172125.116957:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7580. 00010000:00000010:0.1:1786172125.471844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12c9a400. 00010000:00000010:0.1:1786172125.471933:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d1dd500. 00010000:00000010:0.1:1786172125.471939:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261af000. 00000080:00000010:0.1:1786172125.496278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e258c60e0. 00000080:00000010:0.1:1786172125.662427:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15cfa940. 00010000:00000010:0.1:1786172125.662464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e06093a00. 00010000:00000010:0.1:1786172125.662471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa500. 00010000:00000010:0.1:1786172125.662475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e092f5800. 00000080:00000010:0.1:1786172125.801442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e258c6f20. 00000080:00000010:0.1:1786172125.805608:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15ce2100. 00010000:00000010:0.1:1786172125.805630:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12c9ac00. 00010000:00000010:0.1:1786172125.805651:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa700. 00010000:00000010:0.1:1786172125.805656:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e092f4a00. 00000080:00000010:0.1:1786172126.397424:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1193a100. 00000080:00000010:0.1:1786172126.397449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf640. 00000080:00000010:0.1:1786172126.707914:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069780. 00000080:00000010:0.1:1786172126.707949:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13eb39c0. 00010000:00000010:0.1:1786172126.707974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26202800. 00010000:00000010:0.1:1786172126.707979:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418100. 00010000:00000010:0.1:1786172126.707983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d8ff000. 00010000:00000010:0.1:1786172126.805404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26202e00. 00010000:00000010:0.1:1786172126.805423:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02f48400. 00010000:00000010:0.1:1786172126.805428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39cbf000. 00000080:00000010:0.1:1786172127.050817:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a31d280. 00010000:00000010:0.1:1786172127.447684:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2dc00. 00010000:00000010:0.1:1786172127.447698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d800. 00010000:00000010:0.1:1786172127.447703:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b600. 00010000:00000010:0.1:1786172127.447707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31788400. 00000080:00000010:0.1:1786172127.464613:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15cdd280. 00000080:00000010:0.1:1786172127.464625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776d40. 00010000:00000010:0.1:1786172127.474608:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00000080:00000010:0.1:1786172127.672545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7c80. 00010000:00000010:0.1:1786172127.728027:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d83800. 00010000:00000010:0.1:1786172127.728035:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e3ceb00. 00010000:00000010:0.1:1786172127.728040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02fb0c00. 00010000:00000010:0.1:1786172127.728043:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02fb0600. 00000080:00000010:0.1:1786172127.731642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776720. 00000080:00000010:0.1:1786172127.731664:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15cdc200. 00000080:00000010:0.1:1786172127.731673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e0921fe00. 00000080:00000010:0.1:1786172127.757867:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15ec9080. 00000080:00000010:0.1:1786172127.757879:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd080. 00010000:00000010:0.1:1786172127.761557:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d82c00. 00010000:00000010:0.1:1786172127.761567:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e3ce100. 00010000:00000010:0.1:1786172127.761571:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f35c00. 00000080:00000010:0.1:1786172127.766450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776ca0. 00010000:00000010:0.1:1786172127.769931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:0.1:1786172127.991624:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d400. 00010000:00000010:0.1:1786172127.991639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974000. 00010000:00000010:0.1:1786172127.991643:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e093b8d00. 00010000:00000010:0.1:1786172127.991647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31789400. 00000080:00000010:0.1:1786172128.007328:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32d31080. 00000080:00000010:0.1:1786172128.007347:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1f00. 00010000:00000010:0.1:1786172128.023685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:0.1:1786172128.145993:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e72f400. 00010000:00000010:0.1:1786172128.146017:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e068c5e00. 00010000:00000010:0.1:1786172128.237285:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0faaca00. 00010000:00000010:0.1:1786172128.237292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31c8ac00. 00010000:00000010:0.1:1786172128.239966:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0938a700. 00010000:00000010:0.1:1786172128.239974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1025a800. 00010000:00000010:0.1:1786172128.261417:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328afd00. 00010000:00000010:0.1:1786172128.261433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e092f5600. 00010000:00000010:0.1:1786172128.272556:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa800. 00010000:00000010:0.1:1786172128.272566:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09310c00. 00010000:00000010:0.1:1786172128.283297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1513f300. 00010000:00000010:0.1:1786172128.283308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d222200. 00010000:00000010:0.1:1786172128.339517:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d1ddd00. 00010000:00000010:0.1:1786172128.339526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09310e00. 00010000:00000010:0.1:1786172128.339538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e12ee2400. 00010000:00000010:0.1:1786172128.339542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261aec00. 00010000:00000010:0.1:1786172128.339545:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0938af00. 00010000:00000010:0.1:1786172128.339555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31610000. 00000080:00000010:0.1:1786172128.756012:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15cdeb40. 00000080:00000010:0.1:1786172128.756031:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a24d280. 00000080:00000010:0.1:1786172128.756037:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32d32940. 00000080:00000010:0.1:1786172128.756044:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e13eb6b40. 00000080:00000010:0.1:1786172128.756049:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092e7bc0. 00000080:00000010:0.1:1786172128.892445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1e40. 00010000:00000010:0.1:1786172128.908157:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ca00. 00010000:00000010:0.1:1786172128.908166:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:0.1:1786172128.908171:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b400. 00010000:00000010:0.1:1786172128.908175:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39cbfa00. 00000080:00000010:0.1:1786172128.917535:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a24a940. 00000080:00000010:0.1:1786172128.917590:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776060. 00010000:00000010:0.1:1786172128.917600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e092fbe00. 00010000:00000010:0.1:1786172128.917606:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a8000. 00010000:00000010:0.1:1786172128.917610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3954d000. 00000080:00000010:0.1:1786172128.932630:0:2385:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e10b58b60. 00000080:00000010:0.1:1786172128.932671:0:2385:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c09080. 00010000:00000010:0.1:1786172128.935612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31610c00. 00010000:00000010:0.1:1786172128.935640:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e51cf00. 00010000:00000010:0.1:1786172128.935649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e361000. 00010000:00000010:0.1:1786172129.041657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10d82000. 00010000:00000010:0.1:1786172129.041674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261af600. 00010000:00000010:0.1:1786172129.041680:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d1dd000. 00010000:00000010:0.1:1786172129.041699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ae800. 00000080:00000010:0.1:1786172129.046423:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15ce0840. 00000080:00000010:0.1:1786172129.046471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e258c6b00. 00010000:00000010:0.1:1786172129.046489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f9f600. 00010000:00000010:0.1:1786172129.046501:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e10b39d00. 00010000:00000010:0.1:1786172129.046509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e12f08600. 00000080:00000010:0.1:1786172129.067626:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf180. 00010000:00000010:0.1:1786172129.067649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e092fb600. 00010000:00000010:0.1:1786172129.067655:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a8800. 00010000:00000010:0.1:1786172129.067671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2507ea00. 00000080:00000010:0.1:1786172129.079060:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e10b583e0. 00000080:00000010:0.1:1786172129.079093:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c0ca40. 00010000:00000010:0.1:1786172129.087605:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3954dc00. 00010000:00000010:0.1:1786172129.087625:0:178189:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0f7900. 00010000:00000010:0.1:1786172129.087630:0:178189:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e08d49600. 00000080:00000010:0.1:1786172129.144096:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f0a940. 00000080:00000010:0.1:1786172129.144118:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11998840. 00000080:00000010:0.1:1786172129.144125:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e095c1080. 00000080:00000010:0.1:1786172129.144133:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c1b180. 00000080:00000010:0.1:1786172129.144138:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1199d280. 00000080:00000010:0.1:1786172129.144163:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09435ac0. 00000080:00000010:0.1:1786172129.144183:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e095c2940. 00000080:00000010:0.1:1786172129.144188:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c0a940. 00000080:00000010:0.1:1786172129.144200:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11998000. 00000080:00000010:0.1:1786172129.144204:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1199ca40. 00000080:00000010:0.1:1786172129.144215:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c198c0. 00000080:00000010:0.1:1786172129.144228:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e118f18c0. 00000080:00000010:0.1:1786172129.144239:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f09080. 00010000:00000010:0.1:1786172131.438994:0:178351:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e07a9f900. 00010000:00000010:0.1:1786172131.439006:0:178351:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02fa5c00. 00000080:00000010:0.1:1786172132.756472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e04efef40. 00000080:00000010:0.1:1786172132.756481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb57c0. 00000080:00000010:0.1:1786172132.756487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcafa0. 00000080:00000010:0.1:1786172132.756497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e26136b60. 00000080:00000010:0.1:1786172132.756501:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5900. 00000080:00000010:0.1:1786172132.756517:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39cb5640. 00000080:00000010:0.1:1786172132.756522:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e055f9fe0. 00010000:00000010:0.1:1786172143.000472:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401800. 00010000:00000010:0.1:1786172143.013190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9900. 00010000:00000010:0.1:1786172143.013203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0939e200. 00010000:00000010:0.1:1786172161.086562:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:0.1:1786172161.086568:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:0.1:1786172161.100686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:0.1:1786172161.110328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00000080:00000010:0.1:1786172161.238842:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092969c0. 00010000:00000010:0.1:1786172162.986387:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa600. 00010000:00000010:0.1:1786172163.090553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2de00. 00010000:00000010:0.1:1786172163.297916:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c200. 00010000:00000010:0.1:1786172163.804383:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172163.817194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542c00. 00010000:00000010:0.1:1786172163.817213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:0.1:1786172164.079463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:0.1:1786172164.141095:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:0.1:1786172164.343438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:0.1:1786172165.002080:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f400. 00010000:00000010:0.1:1786172165.004306:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:0.1:1786172165.031986:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aac00. 00010000:00000010:0.1:1786172165.032001:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542a00. 00010000:00000010:0.1:1786172165.036107:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c200. 00010000:00000010:0.1:1786172165.037401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527c200. 00010000:00000010:0.1:1786172165.037409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e000. 00010000:00000010:0.1:1786172165.050672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:0.1:1786172165.140408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543600. 00010000:00000010:0.1:1786172165.237490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527dc00. 00010000:00000010:0.1:1786172165.237501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fe00. 00010000:00000010:0.1:1786172165.237507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:0.1:1786172165.237512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543c00. 00010000:00000010:0.1:1786172165.316569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abc00. 00010000:00000010:0.1:1786172165.358804:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e800. 00010000:00000010:0.1:1786172165.390789:0:1:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d800. 00010000:00000010:0.1:1786172165.409670:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d000. 00010000:00000010:0.1:1786172165.442685:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2da00. 00010000:00000010:0.1:1786172165.507093:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172165.616485:0:178205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e600. 00010000:00000010:0.1:1786172165.632439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974600. 00010000:00000010:0.1:1786172165.789009:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d800. 00010000:00000010:0.1:1786172165.838978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d400. 00010000:00000010:0.1:1786172165.838996:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:0.1:1786172165.931989:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527cc00. 00010000:00000010:0.1:1786172166.027492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:0.1:1786172166.148463:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aac00. 00010000:00000010:0.1:1786172166.185240:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:0.1:1786172166.222377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d400. 00010000:00000010:0.1:1786172166.512766:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c600. 00010000:00000010:0.1:1786172166.830147:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172167.053013:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2600. 00010000:00000010:0.1:1786172167.058856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527da00. 00010000:00000010:0.1:1786172167.128499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172167.202393:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2e00. 00010000:00000010:0.1:1786172167.405343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ee00. 00010000:00000010:0.1:1786172167.557153:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543c00. 00010000:00000010:0.1:1786172167.693028:0:152395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fc00. 00010000:00000010:0.1:1786172167.702164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543800. 00010000:00000010:0.1:1786172167.702180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543200. 00010000:00000010:0.1:1786172167.715498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2400. 00010000:00000010:0.1:1786172167.954739:0:178966:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:0.1:1786172168.118006:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3e00. 00010000:00000010:0.1:1786172168.118020:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2200. 00010000:00000010:0.1:1786172168.264343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e000. 00010000:00000010:0.1:1786172168.392166:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172168.402497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2e00. 00010000:00000010:0.1:1786172168.433384:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2c00. 00010000:00000010:0.1:1786172168.509476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ea00. 00010000:00000010:0.1:1786172168.601920:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527c000. 00010000:00000010:0.1:1786172168.645394:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975400. 00010000:00000010:0.1:1786172168.657620:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa600. 00010000:00000010:0.1:1786172168.678450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6c000. 00010000:00000010:0.1:1786172168.816510:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2400. 00010000:00000010:0.1:1786172168.868322:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ec00. 00010000:00000010:0.1:1786172169.104772:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543800. 00010000:00000010:0.1:1786172169.104779:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:0.1:1786172169.104789:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:0.1:1786172169.131445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542400. 00010000:00000010:0.1:1786172169.391395:0:178966:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6dc00. 00010000:00000010:0.1:1786172169.458276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca800. 00010000:00000010:0.1:1786172169.523159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527c400. 00010000:00000010:0.1:1786172169.621483:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:0.1:1786172169.668991:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca400. 00010000:00000010:0.1:1786172169.708378:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb200. 00010000:00000010:0.1:1786172169.729481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527c000. 00010000:00000010:0.1:1786172169.822426:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:0.1:1786172170.095383:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:0.1:1786172170.117086:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ce00. 00010000:00000010:0.1:1786172170.252071:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d400. 00010000:00000010:0.1:1786172170.273649:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:0.1:1786172170.433901:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786172170.521132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:0.1:1786172170.529445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:0.1:1786172170.739379:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527ca00. 00010000:00000010:0.1:1786172170.771936:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217ce00. 00010000:00000010:0.1:1786172170.810478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:0.1:1786172171.015893:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:0.1:1786172171.788704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:0.1:1786172171.965867:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca000. 00010000:00000010:0.1:1786172172.044385:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:0.1:1786172172.061584:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:0.1:1786172172.131413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd600. 00010000:00000010:0.1:1786172172.197568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943800. 00010000:00000010:0.1:1786172172.266458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:0.1:1786172172.270421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543600. 00010000:00000010:0.1:1786172172.547400:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786172172.614391:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cac00. 00010000:00000010:0.1:1786172172.715382:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f000. 00010000:00000010:0.1:1786172172.734836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543800. 00010000:00000010:0.1:1786172172.741515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:0.1:1786172172.921830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c000. 00010000:00000010:0.1:1786172172.995393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31087000. 00010000:00000010:0.1:1786172173.001608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942600. 00010000:00000010:0.1:1786172173.001619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d200. 00010000:00000010:0.1:1786172173.137859:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:0.1:1786172173.161065:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542c00. 00010000:00000010:0.1:1786172173.208378:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca400. 00010000:00000010:0.1:1786172173.374130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb400. 00010000:00000010:0.1:1786172173.411591:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:0.1:1786172173.445877:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31087c00. 00010000:00000010:0.1:1786172173.595509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542600. 00010000:00000010:0.1:1786172174.307515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d400. 00010000:00000010:0.1:1786172174.348714:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:0.1:1786172174.359394:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6dc00. 00010000:00000010:0.1:1786172174.359399:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1200. 00010000:00000010:0.1:1786172174.367627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6c200. 00010000:00000010:0.1:1786172174.367636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:0.1:1786172174.367639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0e00. 00010000:00000010:0.1:1786172174.367646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9c00. 00010000:00000010:0.1:1786172174.452062:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca800. 00010000:00000010:0.1:1786172174.461295:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:0.1:1786172174.461300:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6da00. 00010000:00000010:0.1:1786172174.541607:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0c00. 00010000:00000010:0.1:1786172174.572053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:0.1:1786172174.583591:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:0.1:1786172174.583601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c600. 00010000:00000010:0.1:1786172174.583605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3000. 00010000:00000010:0.1:1786172174.583613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0a00. 00010000:00000010:0.1:1786172174.583627:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:0.1:1786172174.637389:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d800. 00010000:00000010:0.1:1786172174.886780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:0.1:1786172175.043513:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6d400. 00010000:00000010:0.1:1786172175.075403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6d200. 00010000:00000010:0.1:1786172175.215498:0:178982:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0a00. 00010000:00000010:0.1:1786172175.229391:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6cc00. 00010000:00000010:0.1:1786172175.233481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527cc00. 00010000:00000010:0.1:1786172175.266503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d800. 00010000:00000010:0.1:1786172175.399403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1e00. 00010000:00000010:0.1:1786172175.535664:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6d000. 00010000:00000010:0.1:1786172175.583384:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d400. 00010000:00000010:0.1:1786172175.621388:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6c000. 00010000:00000010:0.1:1786172175.752686:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ca00. 00010000:00000010:0.1:1786172175.781487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1400. 00010000:00000010:0.1:1786172175.821820:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d200. 00010000:00000010:0.1:1786172175.882071:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c000. 00010000:00000010:0.1:1786172175.995158:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:0.1:1786172176.003119:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0a00. 00010000:00000010:0.1:1786172176.067791:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217c200. 00010000:00000010:0.1:1786172176.095407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d200. 00010000:00000010:0.1:1786172176.208726:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0c00. 00010000:00000010:0.1:1786172176.283380:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2cc00. 00010000:00000010:0.1:1786172176.294322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:0.1:1786172176.371420:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:0.1:1786172176.761710:0:178966:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3a00. 00010000:00000010:0.1:1786172176.761728:0:178966:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d851800. 00010000:00000010:0.1:1786172177.064430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d851000. 00010000:00000010:0.1:1786172177.064444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d850e00. 00010000:00000010:0.1:1786172177.439679:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6c800. 00010000:00000010:0.1:1786172177.456517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541a00. 00010000:00000010:0.1:1786172177.504388:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f400. 00010000:00000010:0.1:1786172177.685885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2dc00. 00010000:00000010:0.1:1786172177.758003:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2de00. 00010000:00000010:0.1:1786172177.758013:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c800. 00010000:00000010:0.1:1786172177.847412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540c00. 00010000:00000010:0.1:1786172177.883465:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2da00. 00010000:00000010:0.1:1786172177.883484:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541e00. 00010000:00000010:0.1:1786172178.051390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fa00. 00010000:00000010:0.1:1786172178.051396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541000. 00010000:00000010:0.1:1786172178.296191:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6d800. 00010000:00000010:0.1:1786172178.354970:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d850000. 00010000:00000010:0.1:1786172178.354976:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:0.1:1786172178.354980:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d400. 00010000:00000010:0.1:1786172178.361434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc800. 00010000:00000010:0.1:1786172178.365467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d851600. 00010000:00000010:0.1:1786172178.365474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6d400. 00010000:00000010:0.1:1786172178.365477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527da00. 00010000:00000010:0.1:1786172178.466004:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d850800. 00010000:00000010:0.1:1786172178.535394:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6cc00. 00010000:00000010:0.1:1786172178.688034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1c00. 00010000:00000010:0.1:1786172178.688040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e200. 00010000:00000010:0.1:1786172178.773666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d850000. 00010000:00000010:0.1:1786172178.838987:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527c400. 00010000:00000010:0.1:1786172179.043487:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1800. 00010000:00000010:0.1:1786172179.148067:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:0.1:1786172179.335122:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d000. 00010000:00000010:0.1:1786172179.407158:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:0.1:1786172179.606680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9e00. 00010000:00000010:0.1:1786172179.990538:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f800. 00010000:00000010:0.1:1786172180.046418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:0.1:1786172180.111691:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e600. 00010000:00000010:0.1:1786172180.208924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6d800. 00010000:00000010:0.1:1786172180.264459:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:0.1:1786172180.339531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249600. 00010000:00000010:0.1:1786172180.398641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ce00. 00010000:00000010:0.1:1786172180.552057:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:0.1:1786172180.597459:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d200. 00010000:00000010:0.1:1786172180.597465:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0a00. 00010000:00000010:0.1:1786172180.597468:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217ce00. 00010000:00000010:0.1:1786172180.597474:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:0.1:1786172180.597481:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:0.1:1786172180.720818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541a00. 00010000:00000010:0.1:1786172181.120520:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc400. 00010000:00000010:0.1:1786172181.130569:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd600. 00010000:00000010:0.1:1786172181.270478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249600. 00010000:00000010:0.1:1786172181.310501:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d400. 00010000:00000010:0.1:1786172181.376004:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d850c00. 00010000:00000010:0.1:1786172181.376022:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527c400. 00010000:00000010:0.1:1786172181.619376:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d200. 00010000:00000010:0.1:1786172181.699843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:0.1:1786172181.789213:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f800. 00010000:00000010:0.1:1786172181.868704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:0.1:1786172181.946488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:0.1:1786172182.105931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:0.1:1786172182.105945:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172182.327550:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d850a00. 00010000:00000010:0.1:1786172182.537544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:0.1:1786172182.639537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c400. 00010000:00000010:0.1:1786172182.779103:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172182.786507:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e400. 00010000:00000010:0.1:1786172182.786514:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d800. 00010000:00000010:0.1:1786172182.786517:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541200. 00010000:00000010:0.1:1786172182.798003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974600. 00010000:00000010:0.1:1786172182.922905:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e600. 00010000:00000010:0.1:1786172182.922921:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249600. 00010000:00000010:0.1:1786172183.095043:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:0.1:1786172183.095052:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e400. 00010000:00000010:0.1:1786172183.156652:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3600. 00010000:00000010:0.1:1786172183.242668:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f400. 00010000:00000010:0.1:1786172183.256586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6c000. 00010000:00000010:0.1:1786172183.407084:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2200. 00010000:00000010:0.1:1786172183.605126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542c00. 00010000:00000010:0.1:1786172183.737414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:0.1:1786172183.745704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172183.745715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e000. 00010000:00000010:0.1:1786172183.745718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3200. 00010000:00000010:0.1:1786172183.823536:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0527d400. 00010000:00000010:0.1:1786172183.890767:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2400. 00010000:00000010:0.1:1786172183.991667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc400. 00010000:00000010:0.1:1786172184.150788:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ea00. 00010000:00000010:0.1:1786172184.150806:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa000. 00010000:00000010:0.1:1786172184.150810:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:0.1:1786172184.155497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c600. 00010000:00000010:0.1:1786172184.155517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3800. 00010000:00000010:0.1:1786172184.155520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3400. 00010000:00000010:0.1:1786172184.269869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f200. 00010000:00000010:0.1:1786172184.269887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3000. 00010000:00000010:0.1:1786172184.269897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb400. 00010000:00000010:0.1:1786172184.779885:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:0.1:1786172184.854685:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:0.1:1786172184.876429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:0.1:1786172184.936859:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0000. 00010000:00000010:0.1:1786172185.017879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb600. 00010000:00000010:0.1:1786172185.067390:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:0.1:1786172185.067395:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca600. 00010000:00000010:0.1:1786172185.097390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d851c00. 00010000:00000010:0.1:1786172185.109381:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:0.1:1786172185.205478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3000. 00010000:00000010:0.1:1786172185.213225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:0.1:1786172185.302435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2000. 00010000:00000010:0.1:1786172185.473429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975400. 00010000:00000010:0.1:1786172185.476494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172185.476508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610de00. 00010000:00000010:0.1:1786172185.536254:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2200. 00010000:00000010:0.1:1786172185.714964:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ce00. 00010000:00000010:0.1:1786172185.836404:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610da00. 00010000:00000010:0.1:1786172186.093511:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217ce00. 00010000:00000010:0.1:1786172186.506466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943600. 00010000:00000010:0.1:1786172186.730017:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786172186.835842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:0.1:1786172187.103529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610de00. 00010000:00000010:0.1:1786172187.250394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbe00. 00010000:00000010:0.1:1786172187.462443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:0.1:1786172187.791943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:0.1:1786172188.545964:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786172188.778995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:0.1:1786172188.810954:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:0.1:1786172188.816813:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542c00. 00010000:00000010:0.1:1786172189.181567:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f200. 00010000:00000010:0.1:1786172189.266152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942600. 00010000:00000010:0.1:1786172189.292674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abc00. 00010000:00000010:0.1:1786172189.457815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943800. 00010000:00000010:0.1:1786172189.562514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e600. 00010000:00000010:0.1:1786172189.568404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:0.1:1786172189.589403:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:0.1:1786172189.651104:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ea00. 00010000:00000010:0.1:1786172189.806609:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:0.1:1786172189.806630:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ce00. 00010000:00000010:0.1:1786172189.902008:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943400. 00010000:00000010:0.1:1786172190.005210:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d850600. 00010000:00000010:0.1:1786172190.005216:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:0.1:1786172190.005222:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:0.1:1786172190.601403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbe00. 00010000:00000010:0.1:1786172190.601409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d400. 00010000:00000010:0.1:1786172190.601412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6de00. 00010000:00000010:0.1:1786172190.604464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:0.1:1786172190.604474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:0.1:1786172190.604477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:0.1:1786172190.604479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abc00. 00010000:00000010:0.1:1786172190.657414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:0.1:1786172191.023570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c200. 00010000:00000010:0.1:1786172191.570414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541200. 00010000:00000010:0.1:1786172191.729901:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172191.992064:0:178982:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540800. 00010000:00000010:0.1:1786172192.004457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541e00. 00010000:00000010:0.1:1786172192.113499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542a00. 00010000:00000010:0.1:1786172192.142912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:0.1:1786172192.153381:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392a00. 00010000:00000010:0.1:1786172192.269903:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974600. 00010000:00000010:0.1:1786172192.269910:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172192.269914:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb000. 00010000:00000010:0.1:1786172192.284497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6c200. 00010000:00000010:0.1:1786172192.284505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392800. 00010000:00000010:0.1:1786172192.284512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540e00. 00010000:00000010:0.1:1786172192.315488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786172192.315496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d200. 00010000:00000010:0.1:1786172192.536387:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:0.1:1786172192.626411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1200. 00010000:00000010:0.1:1786172192.626425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6da00. 00010000:00000010:0.1:1786172192.745399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2de00. 00010000:00000010:0.1:1786172193.061711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2cc00. 00010000:00000010:0.1:1786172193.061722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392400. 00010000:00000010:0.1:1786172193.402231:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943200. 00010000:00000010:0.1:1786172193.808300:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089ea00. 00010000:00000010:0.1:1786172193.814594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d850400. 00010000:00000010:0.1:1786172193.929429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1200. 00010000:00000010:0.1:1786172194.049934:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:0.1:1786172194.077013:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f400. 00010000:00000010:0.1:1786172194.232468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1400. 00010000:00000010:0.1:1786172194.232478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0800. 00010000:00000010:0.1:1786172194.503737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942400. 00010000:00000010:0.1:1786172194.573408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943600. 00010000:00000010:0.1:1786172194.645076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32afe600. 00010000:00000010:0.1:1786172194.779312:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393a00. 00010000:00000010:0.1:1786172194.880117:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:0.1:1786172195.007608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc200. 00010000:00000010:0.1:1786172195.007634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:0.1:1786172195.021003:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32affa00. 00010000:00000010:0.1:1786172195.139868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1c00. 00010000:00000010:0.1:1786172195.247890:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392800. 00010000:00000010:0.1:1786172195.323842:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943200. 00010000:00000010:0.1:1786172195.515614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32affc00. 00010000:00000010:0.1:1786172195.700413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:0.1:1786172195.760784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fa00. 00010000:00000010:0.1:1786172196.122433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541400. 00010000:00000010:0.1:1786172196.178446:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:0.1:1786172196.480394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:0.1:1786172197.611887:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:0.1:1786172197.782727:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540400. 00010000:00000010:0.1:1786172197.782812:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:0.1:1786172197.782821:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541800. 00010000:00000010:0.1:1786172197.782825:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:0.1:1786172197.799409:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172197.812499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392800. 00010000:00000010:0.1:1786172197.982418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392800. 00010000:00000010:0.1:1786172197.982423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fc00. 00010000:00000010:0.1:1786172197.982430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1c00. 00010000:00000010:0.1:1786172198.331336:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392a00. 00010000:00000010:0.1:1786172198.467858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541000. 00010000:00000010:0.1:1786172198.632878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393000. 00010000:00000010:0.1:1786172198.632887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541a00. 00010000:00000010:0.1:1786172198.632897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1c00. 00010000:00000010:0.1:1786172198.632923:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f800. 00010000:00000010:0.1:1786172198.657405:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089fe00. 00010000:00000010:0.1:1786172198.727415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393800. 00010000:00000010:0.1:1786172198.843754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e000. 00010000:00000010:0.1:1786172198.988225:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089fa00. 00010000:00000010:0.1:1786172199.124759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0a00. 00010000:00000010:0.1:1786172199.124775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540c00. 00010000:00000010:0.1:1786172199.124784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:0.1:1786172199.124788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393400. 00010000:00000010:0.1:1786172199.448895:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:0.1:1786172199.448919:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172199.981642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:0.1:1786172200.111303:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e000. 00010000:00000010:0.1:1786172200.211452:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32aff400. 00010000:00000010:0.1:1786172200.211457:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393000. 00010000:00000010:0.1:1786172200.593455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172201.188389:0:178967:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:0.1:1786172201.213435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:0.1:1786172201.480454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:0.1:1786172201.526421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:0.1:1786172201.583384:0:178967:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cac00. 00010000:00000010:0.1:1786172201.630020:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393600. 00010000:00000010:0.1:1786172201.634846:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123200. 00010000:00000010:0.1:1786172201.821096:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e200. 00010000:00000010:0.1:1786172201.879552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:0.1:1786172202.268188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122de00. 00010000:00000010:0.1:1786172202.334401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00010000:00000010:0.1:1786172202.620541:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32aff200. 00010000:00000010:0.1:1786172202.620561:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca600. 00010000:00000010:0.1:1786172202.651405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:0.1:1786172202.656626:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089ea00. 00010000:00000010:0.1:1786172202.723539:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217c200. 00010000:00000010:0.1:1786172202.918869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d600. 00010000:00000010:0.1:1786172202.918877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:0.1:1786172202.990451:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e400. 00010000:00000010:0.1:1786172202.990457:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541e00. 00010000:00000010:0.1:1786172203.002388:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:0.1:1786172203.002395:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172203.002399:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32affc00. 00010000:00000010:0.1:1786172203.078398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2e00. 00010000:00000010:0.1:1786172203.078404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:0.1:1786172203.167465:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8000. 00010000:00000010:0.1:1786172203.198378:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:0.1:1786172203.398573:0:178992:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610c800. 00010000:00000010:0.1:1786172203.405597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123c00. 00010000:00000010:0.1:1786172203.552787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1200. 00010000:00000010:0.1:1786172203.578737:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:0.1:1786172203.755378:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974a00. 00010000:00000010:0.1:1786172203.890340:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172204.220447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb000. 00010000:00000010:0.1:1786172204.220454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786172204.220469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0600. 00010000:00000010:0.1:1786172204.220473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540c00. 00010000:00000010:0.1:1786172204.220477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:0.1:1786172204.220481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e200. 00010000:00000010:0.1:1786172204.220484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122800. 00010000:00000010:0.1:1786172204.220503:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:0.1:1786172204.231425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:0.1:1786172204.265392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f000. 00010000:00000010:0.1:1786172204.269420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540800. 00010000:00000010:0.1:1786172204.416863:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:0.1:1786172204.500118:0:178994:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123400. 00010000:00000010:0.1:1786172204.500136:0:178994:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb200. 00010000:00000010:0.1:1786172204.508442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:0.1:1786172204.508455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9200. 00010000:00000010:0.1:1786172204.720276:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d400. 00010000:00000010:0.1:1786172204.877435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:0.1:1786172204.877449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aac00. 00010000:00000010:0.1:1786172204.877457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:0.1:1786172204.992861:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6d600. 00010000:00000010:0.1:1786172205.006122:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e400. 00010000:00000010:0.1:1786172205.019381:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f200. 00010000:00000010:0.1:1786172205.146598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e600. 00010000:00000010:0.1:1786172205.269204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb200. 00010000:00000010:0.1:1786172205.443391:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6d600. 00010000:00000010:0.1:1786172205.443396:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0800. 00010000:00000010:0.1:1786172205.539401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1e00. 00010000:00000010:0.1:1786172205.832403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786172205.890742:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540a00. 00010000:00000010:0.1:1786172205.935160:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1600. 00010000:00000010:0.1:1786172206.006514:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610da00. 00010000:00000010:0.1:1786172206.073441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1800. 00010000:00000010:0.1:1786172206.178102:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6d000. 00010000:00000010:0.1:1786172206.178108:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32aff400. 00010000:00000010:0.1:1786172206.178111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541400. 00010000:00000010:0.1:1786172206.178115:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f800. 00010000:00000010:0.1:1786172206.178118:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3800. 00010000:00000010:0.1:1786172206.367406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541000. 00010000:00000010:0.1:1786172206.367411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6d400. 00010000:00000010:0.1:1786172206.367414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393a00. 00010000:00000010:0.1:1786172206.421415:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:0.1:1786172206.456408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942a00. 00010000:00000010:0.1:1786172206.488560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393c00. 00010000:00000010:0.1:1786172206.526373:0:178991:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:0.1:1786172206.541387:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786172206.768005:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c800. 00010000:00000010:0.1:1786172206.911393:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:0.1:1786172206.911403:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:0.1:1786172206.911410:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca800. 00010000:00000010:0.1:1786172206.939088:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1200. 00010000:00000010:0.1:1786172207.058851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0600. 00010000:00000010:0.1:1786172207.223121:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1e00. 00010000:00000010:0.1:1786172207.223137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786172207.297417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:0.1:1786172207.405635:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543c00. 00010000:00000010:0.1:1786172207.563485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392600. 00010000:00000010:0.1:1786172207.615383:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:0.1:1786172207.666395:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:0.1:1786172207.766800:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393400. 00010000:00000010:0.1:1786172207.787456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:0.1:1786172208.033391:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543400. 00010000:00000010:0.1:1786172208.200322:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:0.1:1786172208.275393:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:0.1:1786172208.366956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392e00. 00010000:00000010:0.1:1786172208.446833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:0.1:1786172208.494260:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975400. 00010000:00000010:0.1:1786172208.592751:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:0.1:1786172208.722452:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd200. 00010000:00000010:0.1:1786172208.751461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e800. 00010000:00000010:0.1:1786172208.844440:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e600. 00010000:00000010:0.1:1786172208.856410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fc00. 00010000:00000010:0.1:1786172208.886397:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:0.1:1786172209.284695:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:0.1:1786172209.733765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:0.1:1786172209.990074:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:0.1:1786172209.990081:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fcc00. 00010000:00000010:0.1:1786172210.177460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f200. 00010000:00000010:0.1:1786172210.338919:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:0.1:1786172210.548379:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb000. 00010000:00000010:0.1:1786172210.672383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fe00. 00010000:00000010:0.1:1786172210.672392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543600. 00010000:00000010:0.1:1786172210.860969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e000. 00010000:00000010:0.1:1786172210.860980:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6c600. 00010000:00000010:0.1:1786172210.860983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542000. 00010000:00000010:0.1:1786172210.892677:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942800. 00010000:00000010:0.1:1786172210.902994:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542400. 00010000:00000010:0.1:1786172211.375434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6cc00. 00010000:00000010:0.1:1786172211.375444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942800. 00010000:00000010:0.1:1786172211.383544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540a00. 00010000:00000010:0.1:1786172211.632665:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540e00. 00010000:00000010:0.1:1786172211.681424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541e00. 00010000:00000010:0.1:1786172211.779402:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942600. 00010000:00000010:0.1:1786172211.779436:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541600. 00010000:00000010:0.1:1786172211.886721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc200. 00010000:00000010:0.1:1786172211.915435:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392200. 00010000:00000010:0.1:1786172211.915445:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:0.1:1786172211.915452:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6d000. 00010000:00000010:0.1:1786172212.066703:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca600. 00010000:00000010:0.1:1786172212.172493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e000. 00010000:00000010:0.1:1786172212.225672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3caa00. 00010000:00000010:0.1:1786172212.535706:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172212.558670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6c800. 00010000:00000010:0.1:1786172212.558677:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fce00. 00010000:00000010:0.1:1786172212.590123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca800. 00010000:00000010:0.1:1786172212.725722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540c00. 00010000:00000010:0.1:1786172212.886076:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc200. 00010000:00000010:0.1:1786172212.886089:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb600. 00010000:00000010:0.1:1786172212.886096:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e600. 00010000:00000010:0.1:1786172212.963758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943e00. 00010000:00000010:0.1:1786172213.127413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb400. 00010000:00000010:0.1:1786172213.254526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f800. 00010000:00000010:0.1:1786172213.254532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3600. 00010000:00000010:0.1:1786172213.254556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:0.1:1786172213.254559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542c00. 00010000:00000010:0.1:1786172213.328406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d800. 00010000:00000010:0.1:1786172213.359559:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb400. 00010000:00000010:0.1:1786172213.466433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089fa00. 00010000:00000010:0.1:1786172213.469511:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943e00. 00010000:00000010:0.1:1786172213.895464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e400. 00010000:00000010:0.1:1786172214.024400:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542000. 00010000:00000010:0.1:1786172214.024404:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540e00. 00010000:00000010:0.1:1786172214.025975:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:0.1:1786172214.062787:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786172214.296294:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974600. 00010000:00000010:0.1:1786172214.296307:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540200. 00010000:00000010:0.1:1786172214.296329:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:0.1:1786172214.440452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:0.1:1786172214.440472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541800. 00010000:00000010:0.1:1786172214.496481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f600. 00010000:00000010:0.1:1786172214.777517:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d400. 00010000:00000010:0.1:1786172214.790506:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fcc00. 00010000:00000010:0.1:1786172215.171543:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32aff800. 00010000:00000010:0.1:1786172215.171554:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc800. 00010000:00000010:0.1:1786172215.360673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32afe000. 00010000:00000010:0.1:1786172215.462927:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8800. 00010000:00000010:0.1:1786172215.478519:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:0.1:1786172215.527555:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:0.1:1786172215.815865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392600. 00010000:00000010:0.1:1786172215.822891:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943200. 00010000:00000010:0.1:1786172215.822898:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943000. 00010000:00000010:0.1:1786172215.887446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540a00. 00010000:00000010:0.1:1786172215.943407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32afe400. 00010000:00000010:0.1:1786172215.943413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:0.1:1786172216.136234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541a00. 00010000:00000010:0.1:1786172216.169554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3800. 00010000:00000010:0.1:1786172216.214987:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d600. 00010000:00000010:0.1:1786172216.321385:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d800. 00010000:00000010:0.1:1786172216.321391:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:0.1:1786172216.329430:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2400. 00010000:00000010:0.1:1786172216.329435:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ca00. 00010000:00000010:0.1:1786172216.477488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089ea00. 00010000:00000010:0.1:1786172216.484483:0:178989:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:0.1:1786172216.484498:0:178989:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786172216.493206:0:152395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540600. 00010000:00000010:0.1:1786172216.493212:0:152395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cac00. 00010000:00000010:0.1:1786172216.493218:0:152395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786172216.494834:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543400. 00010000:00000010:0.1:1786172216.557383:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9000. 00010000:00000010:0.1:1786172216.686226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541a00. 00010000:00000010:0.1:1786172216.800638:0:178983:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:0.1:1786172216.849483:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610dc00. 00010000:00000010:0.1:1786172216.849498:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610c600. 00010000:00000010:0.1:1786172216.876379:0:178989:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:0.1:1786172216.876394:0:178989:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786172216.876399:0:178989:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610da00. 00010000:00000010:0.1:1786172216.876406:0:178989:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3c00. 00010000:00000010:0.1:1786172216.890456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217c200. 00010000:00000010:0.1:1786172216.890473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543600. 00010000:00000010:0.1:1786172216.890491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943400. 00010000:00000010:0.1:1786172216.956240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122800. 00010000:00000010:0.1:1786172217.055238:0:152396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32afe800. 00010000:00000010:0.1:1786172217.087527:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca000. 00010000:00000010:0.1:1786172217.087536:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32afe200. 00010000:00000010:0.1:1786172217.196174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172217.256403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:0.1:1786172217.319220:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb000. 00010000:00000010:0.1:1786172217.340475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:0.1:1786172217.383316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:0.1:1786172217.474394:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541a00. 00010000:00000010:0.1:1786172217.560687:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32afe200. 00010000:00000010:0.1:1786172217.602372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:0.1:1786172217.689402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32afea00. 00010000:00000010:0.1:1786172217.721972:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610c400. 00010000:00000010:0.1:1786172217.721983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32afe800. 00010000:00000010:0.1:1786172217.787739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d600. 00010000:00000010:0.1:1786172217.873397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172217.928847:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:0.1:1786172217.969678:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:0.1:1786172218.040388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122de00. 00010000:00000010:0.1:1786172218.062385:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942a00. 00010000:00000010:0.1:1786172218.184396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d800. 00010000:00000010:0.1:1786172218.186399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:0.1:1786172218.551947:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32afe200. 00010000:00000010:0.1:1786172218.611452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610dc00. 00010000:00000010:0.1:1786172218.637425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943000. 00010000:00000010:0.1:1786172218.821583:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:0.1:1786172218.833714:0:178975:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e000. 00010000:00000010:0.1:1786172219.170555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e000. 00010000:00000010:0.1:1786172219.265875:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa000. 00010000:00000010:0.1:1786172219.566546:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542400. 00010000:00000010:0.1:1786172219.675680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cac00. 00010000:00000010:0.1:1786172220.615320:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541600. 00010000:00000010:0.1:1786172220.855443:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e200. 00010000:00000010:0.1:1786172221.071542:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:0.1:1786172221.230386:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e000. 00010000:00000010:0.1:1786172221.295770:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089ea00. 00010000:00000010:0.1:1786172221.404421:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c400. 00010000:00000010:0.1:1786172221.484882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786172221.488456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3caa00. 00010000:00000010:0.1:1786172221.488465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123800. 00010000:00000010:0.1:1786172221.488470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d200. 00010000:00000010:0.1:1786172221.822600:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393400. 00010000:00000010:0.1:1786172221.850104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:0.1:1786172221.905412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786172221.905428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:0.1:1786172221.962885:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392a00. 00010000:00000010:0.1:1786172222.107628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c800. 00010000:00000010:0.1:1786172222.107656:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:0.1:1786172222.107660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:0.1:1786172222.146394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:0.1:1786172222.206492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:0.1:1786172222.206504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:0.1:1786172222.321594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:0.1:1786172222.579419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123c00. 00010000:00000010:0.1:1786172222.832413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c800. 00010000:00000010:0.1:1786172223.345852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f000. 00010000:00000010:0.1:1786172224.037235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0200. 00010000:00000010:0.1:1786172224.320852:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:0.1:1786172224.918384:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32affe00. 00010000:00000010:0.1:1786172224.918389:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2dc00. 00010000:00000010:0.1:1786172225.380377:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1000. 00010000:00000010:0.1:1786172226.355065:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa000. 00010000:00000010:0.1:1786172226.643562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943000. 00010000:00000010:0.1:1786172226.797569:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943200. 00010000:00000010:0.1:1786172227.115094:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392000. 00010000:00000010:0.1:1786172227.145186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbe00. 00010000:00000010:0.1:1786172227.276147:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cac00. 00010000:00000010:0.1:1786172227.893110:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aac00. 00010000:00000010:0.1:1786172228.741540:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943c00. 00010000:00000010:0.1:1786172229.087385:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:0.1:1786172229.104346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2600. 00010000:00000010:0.1:1786172229.625685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32afe200. 00010000:00000010:0.1:1786172229.787560:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:0.1:1786172229.787579:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786172229.887970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3a00. 00010000:00000010:0.1:1786172230.086859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610c800. 00010000:00000010:0.1:1786172230.369342:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9c00. 00010000:00000010:0.1:1786172230.550002:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:0.1:1786172231.045338:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610da00. 00010000:00000010:0.1:1786172231.344225:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2a00. 00010000:00000010:0.1:1786172231.454379:0:178967:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f600. 00010000:00000010:0.1:1786172231.878395:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ec00. 00010000:00000010:0.1:1786172232.322395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d000. 00010000:00000010:0.1:1786172232.329613:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e400. 00010000:00000010:0.1:1786172232.719820:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aac00. 00010000:00000010:0.1:1786172232.967412:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d400. 00010000:00000010:0.1:1786172233.173562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:0.1:1786172233.391578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1e00. 00010000:00000010:0.1:1786172233.910391:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c800. 00010000:00000010:0.1:1786172233.951192:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d600. 00010000:00000010:0.1:1786172234.168800:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9000. 00010000:00000010:0.1:1786172234.168808:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543400. 00010000:00000010:0.1:1786172234.203965:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb800. 00010000:00000010:0.1:1786172234.203972:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393800. 00010000:00000010:0.1:1786172234.203975:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541400. 00010000:00000010:0.1:1786172234.203978:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8600. 00010000:00000010:0.1:1786172234.203982:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089ec00. 00010000:00000010:0.1:1786172234.203985:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd400. 00010000:00000010:0.1:1786172234.203989:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:0.1:1786172234.204038:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123c00. 00010000:00000010:0.1:1786172234.204047:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943200. 00010000:00000010:0.1:1786172234.204053:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172234.204057:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122200. 00010000:00000010:0.1:1786172234.204061:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392000. 00010000:00000010:0.1:1786172234.204065:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392800. 00010000:00000010:0.1:1786172234.390964:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:0.1:1786172234.761945:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:0.1:1786172234.867548:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540800. 00010000:00000010:0.1:1786172235.041334:0:178205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392000. 00010000:00000010:0.1:1786172235.074740:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392200. 00010000:00000010:0.1:1786172235.074745:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:0.1:1786172235.136579:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8a00. 00010000:00000010:0.1:1786172235.445590:0:178966:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540800. 00010000:00000010:0.1:1786172235.713638:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8c00. 00010000:00000010:0.1:1786172235.724405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f000. 00010000:00000010:0.1:1786172236.139268:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2400. 00010000:00000010:0.1:1786172236.152284:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2a00. 00010000:00000010:0.1:1786172236.267990:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f800. 00010000:00000010:0.1:1786172236.686915:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00010000:00000010:0.1:1786172237.213995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3c00. 00010000:00000010:0.1:1786172237.359174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541600. 00010000:00000010:0.1:1786172237.399690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d600. 00010000:00000010:0.1:1786172237.533474:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:0.1:1786172237.706114:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca600. 00010000:00000010:0.1:1786172237.706121:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8a00. 00010000:00000010:0.1:1786172237.733387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:0.1:1786172238.199520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540200. 00010000:00000010:0.1:1786172238.381204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943200. 00010000:00000010:0.1:1786172238.552490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:0.1:1786172238.663523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:0.1:1786172238.741552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392e00. 00010000:00000010:0.1:1786172238.831646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:0.1:1786172238.971127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610c600. 00010000:00000010:0.1:1786172238.988802:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fe00. 00010000:00000010:0.1:1786172239.172389:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d400. 00010000:00000010:0.1:1786172239.256567:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541600. 00010000:00000010:0.1:1786172239.326086:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786172239.553369:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540200. 00010000:00000010:0.1:1786172239.661420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:0.1:1786172239.725266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e000. 00010000:00000010:0.1:1786172239.740540:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca000. 00010000:00000010:0.1:1786172239.911382:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d800. 00010000:00000010:0.1:1786172239.911388:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786172239.914426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172239.994158:0:178990:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:0.1:1786172240.015382:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541600. 00010000:00000010:0.1:1786172240.057395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cae00. 00010000:00000010:0.1:1786172240.236076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f400. 00010000:00000010:0.1:1786172240.372693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:0.1:1786172240.384033:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943800. 00010000:00000010:0.1:1786172240.384040:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cba00. 00010000:00000010:0.1:1786172240.518392:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00010000:00000010:0.1:1786172240.718754:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3e00. 00010000:00000010:0.1:1786172240.800878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb200. 00010000:00000010:0.1:1786172240.800886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3000. 00010000:00000010:0.1:1786172240.800889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786172241.113022:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d600. 00010000:00000010:0.1:1786172241.490234:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3200. 00010000:00000010:0.1:1786172241.521517:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2000. 00010000:00000010:0.1:1786172241.641722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2200. 00010000:00000010:0.1:1786172241.744393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e000. 00010000:00000010:0.1:1786172241.744402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089ee00. 00010000:00000010:0.1:1786172241.744406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1e00. 00010000:00000010:0.1:1786172241.837693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:0.1:1786172241.914308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:0.1:1786172242.234393:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542a00. 00010000:00000010:0.1:1786172242.277488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:0.1:1786172242.352723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:0.1:1786172242.352729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f800. 00010000:00000010:0.1:1786172242.549857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122e00. 00010000:00000010:0.1:1786172242.549865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2400. 00010000:00000010:0.1:1786172242.654383:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cba00. 00010000:00000010:0.1:1786172242.668469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:0.1:1786172242.705040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:0.1:1786172242.714450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:0.1:1786172242.729664:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172242.997402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3800. 00010000:00000010:0.1:1786172243.034663:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:0.1:1786172243.053240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb000. 00010000:00000010:0.1:1786172243.068616:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:0.1:1786172243.288087:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:0.1:1786172243.442001:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3e00. 00010000:00000010:0.1:1786172243.517399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393e00. 00010000:00000010:0.1:1786172243.640493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392a00. 00010000:00000010:0.1:1786172243.640499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:0.1:1786172243.653096:0:178995:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1000. 00010000:00000010:0.1:1786172243.731382:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393800. 00010000:00000010:0.1:1786172243.855412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943c00. 00010000:00000010:0.1:1786172243.903898:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:0.1:1786172243.923376:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c000. 00010000:00000010:0.1:1786172243.923382:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00010000:00000010:0.1:1786172243.953394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:0.1:1786172243.953400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e200. 00010000:00000010:0.1:1786172243.953403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f400. 00010000:00000010:0.1:1786172244.119015:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2da00. 00010000:00000010:0.1:1786172244.161427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2a00. 00010000:00000010:0.1:1786172244.177409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2c00. 00010000:00000010:0.1:1786172244.376038:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:0.1:1786172244.518380:0:178993:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f400. 00010000:00000010:0.1:1786172244.518386:0:178993:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f000. 00010000:00000010:0.1:1786172244.558806:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089ee00. 00010000:00000010:0.1:1786172244.619463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ca00. 00010000:00000010:0.1:1786172244.734385:0:178993:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393600. 00010000:00000010:0.1:1786172244.749664:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:0.1:1786172245.141419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aaa00. 00010000:00000010:0.1:1786172245.276483:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:0.1:1786172245.477443:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786172245.631410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec2a00. 00010000:00000010:0.1:1786172245.836382:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e200. 00010000:00000010:0.1:1786172245.836391:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d400. 00010000:00000010:0.1:1786172245.836397:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc200. 00010000:00000010:0.1:1786172245.909425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec2000. 00010000:00000010:0.1:1786172246.127602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c400. 00010000:00000010:0.1:1786172246.177869:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:0.1:1786172246.177874:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817600. 00010000:00000010:0.1:1786172246.224417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:0.1:1786172246.385409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca800. 00010000:00000010:0.1:1786172246.463476:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:0.1:1786172246.468420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3600. 00010000:00000010:0.1:1786172246.468438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e3200. 00010000:00000010:0.1:1786172246.470699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:0.1:1786172246.530526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:0.1:1786172246.734010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393200. 00010000:00000010:0.1:1786172246.835924:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392800. 00010000:00000010:0.1:1786172246.835944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbe00. 00010000:00000010:0.1:1786172247.112694:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393400. 00010000:00000010:0.1:1786172247.271326:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c000. 00010000:00000010:0.1:1786172247.406739:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393800. 00010000:00000010:0.1:1786172247.854473:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c000. 00010000:00000010:0.1:1786172247.854477:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817800. 00010000:00000010:0.1:1786172248.615824:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393400. 00010000:00000010:0.1:1786172248.712380:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f200. 00010000:00000010:0.1:1786172248.790092:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c000. 00010000:00000010:0.1:1786172248.910770:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540c00. 00010000:00000010:0.1:1786172249.029412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393400. 00010000:00000010:0.1:1786172249.088491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541600. 00010000:00000010:0.1:1786172249.088498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393a00. 00010000:00000010:0.1:1786172249.296387:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e200. 00010000:00000010:0.1:1786172249.415383:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec2a00. 00010000:00000010:0.1:1786172249.420546:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3600. 00010000:00000010:0.1:1786172249.459736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f000. 00010000:00000010:0.1:1786172249.565126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e100e2000. 00010000:00000010:0.1:1786172249.596394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:0.1:1786172249.622687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c000. 00010000:00000010:0.1:1786172249.648444:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec2400. 00010000:00000010:0.1:1786172249.713124:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393800. 00010000:00000010:0.1:1786172249.713129:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:0.1:1786172249.749460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ea00. 00010000:00000010:0.1:1786172249.880413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:0.1:1786172249.950665:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392000. 00010000:00000010:0.1:1786172250.282724:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089ec00. 00010000:00000010:0.1:1786172250.366525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f000. 00010000:00000010:0.1:1786172250.366549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:0.1:1786172250.698484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:0.1:1786172250.736854:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd600. 00010000:00000010:0.1:1786172250.829640:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc800. 00010000:00000010:0.1:1786172250.864412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:0.1:1786172250.864417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122de00. 00010000:00000010:0.1:1786172250.868216:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786172250.877413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:0.1:1786172250.877428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9000. 00010000:00000010:0.1:1786172250.960325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d800. 00010000:00000010:0.1:1786172250.960343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8600. 00010000:00000010:0.1:1786172251.014782:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393e00. 00010000:00000010:0.1:1786172251.014788:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2cc00. 00010000:00000010:0.1:1786172251.014791:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:0.1:1786172251.014794:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd200. 00010000:00000010:0.1:1786172251.014798:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817000. 00010000:00000010:0.1:1786172251.014801:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c400. 00010000:00000010:0.1:1786172251.014809:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540000. 00010000:00000010:0.1:1786172251.014819:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fe00. 00010000:00000010:0.1:1786172251.014824:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172251.061683:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e200. 00010000:00000010:0.1:1786172251.064400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd600. 00010000:00000010:0.1:1786172251.072413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8c00. 00010000:00000010:0.1:1786172251.310198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f800. 00010000:00000010:0.1:1786172251.336622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8200. 00010000:00000010:0.1:1786172251.478439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8600. 00010000:00000010:0.1:1786172251.529562:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:0.1:1786172251.846916:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e800. 00010000:00000010:0.1:1786172251.886428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ec00. 00010000:00000010:0.1:1786172252.012274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9e00. 00010000:00000010:0.1:1786172252.142948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542000. 00010000:00000010:0.1:1786172252.142954:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00010000:00000010:0.1:1786172252.179407:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f400. 00010000:00000010:0.1:1786172252.387599:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610de00. 00010000:00000010:0.1:1786172252.626683:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f000. 00010000:00000010:0.1:1786172252.655182:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:0.1:1786172252.702897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541000. 00010000:00000010:0.1:1786172252.922994:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:0.1:1786172253.079924:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:0.1:1786172253.371388:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0200. 00010000:00000010:0.1:1786172253.697337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942600. 00010000:00000010:0.1:1786172253.748299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8800. 00010000:00000010:0.1:1786172253.905008:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:0.1:1786172253.913125:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1400. 00010000:00000010:0.1:1786172254.062391:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0000. 00010000:00000010:0.1:1786172254.243417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1600. 00010000:00000010:0.1:1786172254.399627:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d000. 00010000:00000010:0.1:1786172254.526475:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d200. 00010000:00000010:0.1:1786172254.541452:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393800. 00010000:00000010:3.1:1786172254.553476:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540600. 00010000:00000010:3.1:1786172254.553481:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0e00. 00010000:00000010:0.1:1786172254.568508:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fe00. 00010000:00000010:3.1:1786172254.621641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540800. 00010000:00000010:3.1:1786172254.647202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172254.649462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786172254.649879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172254.649886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975400. 00010000:00000010:3.1:1786172254.685737:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:3.1:1786172254.685743:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:0.1:1786172254.696435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8200. 00010000:00000010:0.1:1786172254.696441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:0.1:1786172254.696445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393200. 00010000:00000010:0.1:1786172254.696448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:3.1:1786172254.696641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0800. 00010000:00000010:3.1:1786172254.696649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fcc00. 00010000:00000010:3.1:1786172254.696662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1e00. 00010000:00000010:3.1:1786172254.730392:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d800. 00010000:00000010:0.1:1786172254.748459:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:0.1:1786172254.748468:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610c200. 00010000:00000010:3.1:1786172254.751405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e400. 00010000:00000010:3.1:1786172254.765028:0:178195:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089fc00. 00010000:00000010:0.1:1786172254.823764:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:3.1:1786172254.977148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541a00. 00010000:00000010:3.1:1786172255.021214:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610dc00. 00010000:00000010:0.1:1786172255.093154:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f800. 00010000:00000010:3.1:1786172255.109610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393e00. 00010000:00000010:0.1:1786172255.307443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540a00. 00010000:00000010:0.1:1786172255.329725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:3.1:1786172255.363409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:3.1:1786172255.418816:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:3.1:1786172255.469421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:3.1:1786172255.504273:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e800. 00010000:00000010:3.1:1786172255.504280:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:3.1:1786172255.559355:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:0.1:1786172255.603697:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d200. 00010000:00000010:3.1:1786172255.682480:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fce00. 00010000:00000010:3.1:1786172255.682491:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fe00. 00010000:00000010:3.1:1786172255.682495:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540400. 00010000:00000010:3.1:1786172255.705382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc400. 00010000:00000010:3.1:1786172255.760753:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e800. 00010000:00000010:0.1:1786172255.763737:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543e00. 00010000:00000010:3.1:1786172255.805049:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ea00. 00010000:00000010:0.1:1786172255.834731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:0.1:1786172255.864384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd200. 00010000:00000010:0.1:1786172255.878336:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541400. 00010000:00000010:3.1:1786172255.879508:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540200. 00010000:00000010:3.1:1786172255.885453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543600. 00010000:00000010:3.1:1786172255.885464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540a00. 00010000:00000010:0.1:1786172256.003567:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393600. 00010000:00000010:3.1:1786172256.055335:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:3.1:1786172256.149118:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816a00. 00010000:00000010:3.1:1786172256.149136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540e00. 00010000:00000010:3.1:1786172256.149141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123200. 00010000:00000010:3.1:1786172256.149148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172256.149157:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:0.1:1786172256.162445:0:178992:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543800. 00010000:00000010:3.1:1786172256.163579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:3.1:1786172256.163587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943400. 00010000:00000010:0.1:1786172256.170417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc400. 00010000:00000010:3.1:1786172256.213165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:3.1:1786172256.237596:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:3.1:1786172256.273702:0:178205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd600. 00010000:00000010:3.1:1786172256.279394:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541000. 00010000:00000010:3.1:1786172256.315181:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543400. 00010000:00000010:3.1:1786172256.319313:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3400. 00010000:00000010:3.1:1786172256.362778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec2a00. 00010000:00000010:3.1:1786172256.365396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1600. 00010000:00000010:0.1:1786172256.406537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540800. 00010000:00000010:0.1:1786172256.448610:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2de00. 00010000:00000010:3.1:1786172256.504444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610c000. 00010000:00000010:0.1:1786172256.505751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c200. 00010000:00000010:3.1:1786172256.531903:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fce00. 00010000:00000010:3.1:1786172256.549473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e400. 00010000:00000010:3.1:1786172256.555878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c600. 00010000:00000010:3.1:1786172256.561478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d800. 00010000:00000010:3.1:1786172256.599453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543800. 00010000:00000010:0.1:1786172256.672390:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc400. 00010000:00000010:0.1:1786172256.723814:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:3.1:1786172256.755668:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3800. 00010000:00000010:3.1:1786172256.755682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:3.1:1786172256.755689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974a00. 00010000:00000010:0.1:1786172256.756162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:3.1:1786172256.855412:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd400. 00010000:00000010:3.1:1786172256.861931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543600. 00010000:00000010:3.1:1786172256.861948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786172256.960857:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:3.1:1786172257.036814:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0200. 00010000:00000010:3.1:1786172257.036819:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942800. 00010000:00000010:0.1:1786172257.046446:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:3.1:1786172257.103582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:3.1:1786172257.108398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1600. 00010000:00000010:3.1:1786172257.108404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543e00. 00010000:00000010:3.1:1786172257.207309:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1400. 00010000:00000010:0.1:1786172257.310395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:0.1:1786172257.310400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c600. 00010000:00000010:3.1:1786172257.310442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c200. 00010000:00000010:0.1:1786172257.438486:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3400. 00010000:00000010:0.1:1786172257.438510:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543e00. 00010000:00000010:0.1:1786172257.467382:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540c00. 00010000:00000010:0.1:1786172257.467401:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd600. 00010000:00000010:3.1:1786172257.487865:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1400. 00010000:00000010:3.1:1786172257.549505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541600. 00010000:00000010:3.1:1786172257.549511:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f200. 00010000:00000010:0.1:1786172257.550071:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:0.1:1786172257.550079:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3800. 00010000:00000010:0.1:1786172257.550096:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:0.1:1786172257.550099:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:0.1:1786172257.550104:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943e00. 00010000:00000010:0.1:1786172257.550110:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3a00. 00010000:00000010:0.1:1786172257.556550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:0.1:1786172257.556562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:0.1:1786172257.556567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543200. 00010000:00000010:0.1:1786172257.556579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1a00. 00010000:00000010:3.1:1786172257.556779:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:0.1:1786172257.581450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975400. 00010000:00000010:0.1:1786172257.720626:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:3.1:1786172257.727703:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d400. 00010000:00000010:3.1:1786172257.771397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca400. 00010000:00000010:0.1:1786172257.776040:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0600. 00010000:00000010:3.1:1786172257.908452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3e00. 00010000:00000010:3.1:1786172257.955606:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0c00. 00010000:00000010:3.1:1786172258.006388:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172258.006394:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:3.1:1786172258.006397:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542000. 00010000:00000010:0.1:1786172258.093141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb200. 00010000:00000010:0.1:1786172258.094518:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb800. 00010000:00000010:3.1:1786172258.107461:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa600. 00010000:00000010:3.1:1786172258.146486:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cac00. 00010000:00000010:0.1:1786172258.174581:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3800. 00010000:00000010:3.1:1786172258.277487:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb000. 00010000:00000010:3.1:1786172258.282497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3c00. 00010000:00000010:3.1:1786172258.306747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2cc00. 00010000:00000010:0.1:1786172258.344053:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cba00. 00010000:00000010:0.1:1786172258.395026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb200. 00010000:00000010:3.1:1786172258.465404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:0.1:1786172258.474195:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca600. 00010000:00000010:3.1:1786172258.516780:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:3.1:1786172258.516785:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d600. 00010000:00000010:0.1:1786172258.568492:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0a00. 00010000:00000010:0.1:1786172258.568500:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0c00. 00010000:00000010:0.1:1786172258.903724:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:3.1:1786172258.948102:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca200. 00010000:00000010:0.1:1786172258.975462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:3.1:1786172258.975625:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3800. 00010000:00000010:0.1:1786172259.031405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3a00. 00010000:00000010:0.1:1786172259.045458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb400. 00010000:00000010:0.1:1786172259.065855:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca000. 00010000:00000010:3.1:1786172259.082149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ca00. 00010000:00000010:3.1:1786172259.112513:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:0.1:1786172259.112588:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542c00. 00010000:00000010:3.1:1786172259.238267:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb600. 00010000:00000010:0.1:1786172259.238558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:0.1:1786172259.332249:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3200. 00010000:00000010:0.1:1786172259.343388:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1a00. 00010000:00000010:0.1:1786172259.392252:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb200. 00010000:00000010:0.1:1786172259.649013:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1400. 00010000:00000010:0.1:1786172259.766455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cba00. 00010000:00000010:0.1:1786172259.831829:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:0.1:1786172259.831837:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:3.1:1786172259.841830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816200. 00010000:00000010:0.1:1786172259.842407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817000. 00010000:00000010:3.1:1786172259.940387:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d400. 00010000:00000010:3.1:1786172259.940394:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0c00. 00010000:00000010:3.1:1786172260.041489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb600. 00010000:00000010:0.1:1786172260.180200:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca600. 00010000:00000010:0.1:1786172260.180207:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542a00. 00010000:00000010:0.1:1786172260.180210:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb400. 00010000:00000010:0.1:1786172260.202502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123400. 00010000:00000010:0.1:1786172260.202507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:0.1:1786172260.202510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0400. 00010000:00000010:0.1:1786172260.202519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816400. 00010000:00000010:0.1:1786172260.202974:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1000. 00010000:00000010:3.1:1786172260.204394:0:178983:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:3.1:1786172260.204405:0:178983:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817200. 00010000:00000010:3.1:1786172260.225109:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2cc00. 00010000:00000010:3.1:1786172260.267489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2dc00. 00010000:00000010:0.1:1786172260.366069:0:178985:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb200. 00010000:00000010:0.1:1786172260.366076:0:178985:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816e00. 00010000:00000010:3.1:1786172260.367156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3600. 00010000:00000010:3.1:1786172260.367163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:0.1:1786172260.385471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3e00. 00010000:00000010:3.1:1786172260.395387:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1c00. 00010000:00000010:0.1:1786172260.404261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c000. 00010000:00000010:0.1:1786172260.404275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ca00. 00010000:00000010:0.1:1786172260.413245:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543800. 00010000:00000010:0.1:1786172260.413254:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca000. 00010000:00000010:0.1:1786172260.413259:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1600. 00010000:00000010:3.1:1786172260.447764:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec2400. 00010000:00000010:3.1:1786172260.447774:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:3.1:1786172260.447777:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610c800. 00010000:00000010:3.1:1786172260.447784:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec2800. 00010000:00000010:0.1:1786172260.455501:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0000. 00010000:00000010:0.1:1786172260.502286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec2e00. 00010000:00000010:3.1:1786172260.502506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817400. 00010000:00000010:0.1:1786172260.595619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:0.1:1786172260.680527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca600. 00010000:00000010:3.1:1786172260.765075:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1600. 00010000:00000010:0.1:1786172260.812722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122800. 00010000:00000010:0.1:1786172260.922494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0400. 00010000:00000010:0.1:1786172260.959440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ca00. 00010000:00000010:0.1:1786172261.124068:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c200. 00010000:00000010:3.1:1786172261.131957:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1c00. 00010000:00000010:0.1:1786172261.263499:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb400. 00010000:00000010:0.1:1786172261.302413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089ec00. 00010000:00000010:0.1:1786172261.434410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbe00. 00010000:00000010:3.1:1786172261.500043:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543600. 00010000:00000010:3.1:1786172261.523430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2dc00. 00010000:00000010:3.1:1786172261.561785:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d400. 00010000:00000010:0.1:1786172261.607568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cae00. 00010000:00000010:3.1:1786172261.664466:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cba00. 00010000:00000010:3.1:1786172261.670491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:3.1:1786172261.740815:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec3a00. 00010000:00000010:3.1:1786172261.808460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1400. 00010000:00000010:3.1:1786172261.839855:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1e00. 00010000:00000010:3.1:1786172261.878386:0:178994:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2cc00. 00010000:00000010:3.1:1786172261.899383:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:0.1:1786172261.918489:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cba00. 00010000:00000010:0.1:1786172261.954450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aaa00. 00010000:00000010:0.1:1786172261.965244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943200. 00010000:00000010:3.1:1786172262.003629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:3.1:1786172262.073530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0600. 00010000:00000010:3.1:1786172262.073535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2dc00. 00010000:00000010:0.1:1786172262.113440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122e00. 00010000:00000010:3.1:1786172262.156390:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1600. 00010000:00000010:0.1:1786172262.156564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:0.1:1786172262.165794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816e00. 00010000:00000010:3.1:1786172262.184460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:3.1:1786172262.260387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:3.1:1786172262.260392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:3.1:1786172262.260401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f400. 00010000:00000010:3.1:1786172262.335485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0800. 00010000:00000010:3.1:1786172262.387606:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:0.1:1786172262.388584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816200. 00010000:00000010:3.1:1786172262.512381:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:3.1:1786172262.521807:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca000. 00010000:00000010:0.1:1786172262.543447:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca600. 00010000:00000010:3.1:1786172262.560553:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816400. 00010000:00000010:0.1:1786172262.597604:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f000. 00010000:00000010:0.1:1786172262.597622:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e400. 00010000:00000010:3.1:1786172262.604388:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa000. 00010000:00000010:3.1:1786172262.604395:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb000. 00010000:00000010:3.1:1786172262.604399:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816c00. 00010000:00000010:3.1:1786172262.604403:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:3.1:1786172262.619959:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:3.1:1786172262.632163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816a00. 00010000:00000010:3.1:1786172262.640432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393200. 00010000:00000010:3.1:1786172262.754448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817e00. 00010000:00000010:0.1:1786172262.805430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172262.808795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb200. 00010000:00000010:0.1:1786172262.893385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:0.1:1786172262.942866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393200. 00010000:00000010:3.1:1786172262.953404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816800. 00010000:00000010:3.1:1786172263.022377:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:0.1:1786172263.089230:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393a00. 00010000:00000010:0.1:1786172263.089237:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec1200. 00010000:00000010:3.1:1786172263.090098:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c200. 00010000:00000010:3.1:1786172263.090106:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca400. 00010000:00000010:3.1:1786172263.090111:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2de00. 00010000:00000010:3.1:1786172263.090123:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:3.1:1786172263.090129:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:3.1:1786172263.090148:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:3.1:1786172263.090159:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:3.1:1786172263.090169:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393800. 00010000:00000010:3.1:1786172263.090174:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3caa00. 00010000:00000010:0.1:1786172263.093381:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fcc00. 00010000:00000010:0.1:1786172263.093387:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ec00. 00010000:00000010:0.1:1786172263.093402:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ca00. 00010000:00000010:0.1:1786172263.103399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbe00. 00010000:00000010:3.1:1786172263.192827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb600. 00010000:00000010:3.1:1786172263.230561:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817c00. 00010000:00000010:3.1:1786172263.230570:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123200. 00010000:00000010:3.1:1786172263.230575:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2da00. 00010000:00000010:0.1:1786172263.241490:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172263.303789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543400. 00010000:00000010:0.1:1786172263.371650:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:3.1:1786172263.380542:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca400. 00010000:00000010:0.1:1786172263.416445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393600. 00010000:00000010:0.1:1786172263.464423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123c00. 00010000:00000010:3.1:1786172263.509496:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543c00. 00010000:00000010:0.1:1786172263.617698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:0.1:1786172263.733627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975400. 00010000:00000010:0.1:1786172263.816746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3caa00. 00010000:00000010:3.1:1786172263.838025:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974a00. 00010000:00000010:3.1:1786172263.868385:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817600. 00010000:00000010:0.1:1786172264.036442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd400. 00010000:00000010:0.1:1786172264.062657:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392a00. 00010000:00000010:3.1:1786172264.121846:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393000. 00010000:00000010:3.1:1786172264.225430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543e00. 00010000:00000010:3.1:1786172264.331214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172264.423150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393400. 00010000:00000010:0.1:1786172264.426466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:0.1:1786172264.426486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393800. 00010000:00000010:3.1:1786172264.529384:0:178966:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817c00. 00010000:00000010:3.1:1786172264.691401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122e00. 00010000:00000010:3.1:1786172264.815402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb800. 00010000:00000010:3.1:1786172264.847879:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:0.1:1786172264.859166:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca000. 00010000:00000010:3.1:1786172264.913860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:3.1:1786172265.064911:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0a00. 00010000:00000010:0.1:1786172265.107662:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f800. 00010000:00000010:3.1:1786172265.174444:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:0.1:1786172265.266207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816400. 00010000:00000010:0.1:1786172265.266214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543600. 00010000:00000010:0.1:1786172265.296345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540000. 00010000:00000010:0.1:1786172265.296379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172265.318622:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ea00. 00010000:00000010:3.1:1786172265.318631:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca600. 00010000:00000010:3.1:1786172265.327761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb400. 00010000:00000010:0.1:1786172265.370441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb600. 00010000:00000010:3.1:1786172265.370462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393200. 00010000:00000010:3.1:1786172265.370479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc200. 00010000:00000010:3.1:1786172265.566526:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb400. 00010000:00000010:3.1:1786172265.566531:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e600. 00010000:00000010:3.1:1786172265.587589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:3.1:1786172265.657397:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540c00. 00010000:00000010:0.1:1786172265.754889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d600. 00010000:00000010:3.1:1786172265.768419:0:1348:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cae00. 00010000:00000010:3.1:1786172265.911261:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393800. 00010000:00000010:0.1:1786172265.967650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d400. 00010000:00000010:0.1:1786172265.973213:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d000. 00010000:00000010:0.1:1786172265.973219:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541600. 00010000:00000010:3.1:1786172266.163240:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cae00. 00010000:00000010:0.1:1786172266.177578:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392800. 00010000:00000010:0.1:1786172266.177585:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00010000:00000010:0.1:1786172266.266388:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb600. 00010000:00000010:0.1:1786172266.266394:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cba00. 00010000:00000010:0.1:1786172266.266397:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392a00. 00010000:00000010:0.1:1786172266.326457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:0.1:1786172266.392069:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816000. 00010000:00000010:3.1:1786172266.476498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:3.1:1786172266.488435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:3.1:1786172266.512462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:0.1:1786172266.557517:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:0.1:1786172266.557524:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f800. 00010000:00000010:0.1:1786172266.592400:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abc00. 00010000:00000010:3.1:1786172266.671469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb400. 00010000:00000010:3.1:1786172266.776865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540600. 00010000:00000010:3.1:1786172266.776877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393600. 00010000:00000010:3.1:1786172266.776880:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392e00. 00010000:00000010:3.1:1786172266.776883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec0400. 00010000:00000010:3.1:1786172266.776886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:3.1:1786172266.782512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541e00. 00010000:00000010:0.1:1786172266.782667:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02ec2000. 00010000:00000010:0.1:1786172266.782674:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca200. 00010000:00000010:0.1:1786172266.821899:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610c800. 00010000:00000010:0.1:1786172266.821905:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00010000:00000010:3.1:1786172266.843402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540e00. 00010000:00000010:3.1:1786172266.843410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541600. 00010000:00000010:0.1:1786172266.843769:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:0.1:1786172266.927568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393e00. 00010000:00000010:3.1:1786172266.987646:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392a00. 00010000:00000010:0.1:1786172266.991398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:3.1:1786172267.011379:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393c00. 00010000:00000010:3.1:1786172267.011400:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122de00. 00010000:00000010:0.1:1786172267.011818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca400. 00010000:00000010:0.1:1786172267.016454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:0.1:1786172267.053586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392e00. 00010000:00000010:3.1:1786172267.059109:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:0.1:1786172267.247967:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816600. 00010000:00000010:3.1:1786172267.308024:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:3.1:1786172267.308032:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e800. 00010000:00000010:3.1:1786172267.337995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172267.345402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249c00. 00010000:00000010:3.1:1786172267.392821:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393800. 00010000:00000010:0.1:1786172267.407540:0:178993:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ce00. 00010000:00000010:0.1:1786172267.407548:0:178993:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541400. 00010000:00000010:0.1:1786172267.416500:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:0.1:1786172267.459217:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393a00. 00010000:00000010:3.1:1786172267.469661:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610c800. 00010000:00000010:0.1:1786172267.482415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d600. 00010000:00000010:3.1:1786172267.483030:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ea00. 00010000:00000010:0.1:1786172267.614375:0:178986:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816e00. 00010000:00000010:3.1:1786172267.616843:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542600. 00010000:00000010:0.1:1786172267.620333:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:3.1:1786172267.632629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392800. 00010000:00000010:3.1:1786172267.640467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392400. 00010000:00000010:3.1:1786172267.640485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610ce00. 00010000:00000010:0.1:1786172267.654027:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:3.1:1786172267.696517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172267.777725:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393e00. 00010000:00000010:0.1:1786172267.778593:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610dc00. 00010000:00000010:3.1:1786172267.782112:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:0.1:1786172267.800899:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392a00. 00010000:00000010:0.1:1786172267.967447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393200. 00010000:00000010:0.1:1786172267.967453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942400. 00010000:00000010:0.1:1786172268.008408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943600. 00010000:00000010:3.1:1786172268.029001:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d800. 00010000:00000010:3.1:1786172268.033602:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540000. 00010000:00000010:3.1:1786172268.088519:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172268.088539:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540a00. 00010000:00000010:3.1:1786172268.304488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942600. 00010000:00000010:0.1:1786172268.311846:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca000. 00010000:00000010:3.1:1786172268.313671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa600. 00010000:00000010:3.1:1786172268.313684:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:3.1:1786172268.313688:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392600. 00010000:00000010:3.1:1786172268.456047:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943c00. 00010000:00000010:3.1:1786172268.456057:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540a00. 00010000:00000010:0.1:1786172268.494682:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541e00. 00010000:00000010:3.1:1786172268.554581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610da00. 00010000:00000010:0.1:1786172268.582486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943c00. 00010000:00000010:0.1:1786172268.589696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d800. 00010000:00000010:3.1:1786172268.649168:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392600. 00010000:00000010:3.1:1786172268.649176:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540e00. 00010000:00000010:3.1:1786172268.700773:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541800. 00010000:00000010:3.1:1786172268.704471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393200. 00010000:00000010:3.1:1786172268.718426:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00010000:00000010:3.1:1786172268.760398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942400. 00010000:00000010:3.1:1786172268.826492:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942a00. 00010000:00000010:3.1:1786172268.856135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943600. 00010000:00000010:0.1:1786172268.860456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f800. 00010000:00000010:3.1:1786172268.860487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00010000:00000010:3.1:1786172268.955997:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8200. 00010000:00000010:0.1:1786172268.976429:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ce00. 00010000:00000010:3.1:1786172269.003728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f800. 00010000:00000010:3.1:1786172269.096769:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e800. 00010000:00000010:3.1:1786172269.096791:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393200. 00010000:00000010:0.1:1786172269.104386:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e200. 00010000:00000010:0.1:1786172269.104391:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e600. 00010000:00000010:0.1:1786172269.138420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:3.1:1786172269.150396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943000. 00010000:00000010:0.1:1786172269.170473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f200. 00010000:00000010:3.1:1786172269.227565:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392e00. 00010000:00000010:0.1:1786172269.249617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c000. 00010000:00000010:3.1:1786172269.261043:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ea00. 00010000:00000010:3.1:1786172269.281446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d400. 00010000:00000010:3.1:1786172269.298387:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610dc00. 00010000:00000010:0.1:1786172269.336863:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:0.1:1786172269.336883:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d800. 00010000:00000010:3.1:1786172269.373386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610cc00. 00010000:00000010:3.1:1786172269.376405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8000. 00010000:00000010:3.1:1786172269.436425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9000. 00010000:00000010:3.1:1786172269.436440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942400. 00010000:00000010:3.1:1786172269.445966:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392400. 00010000:00000010:0.1:1786172269.498952:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8600. 00010000:00000010:0.1:1786172269.504925:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:3.1:1786172269.529380:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943000. 00010000:00000010:3.1:1786172269.536977:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8c00. 00010000:00000010:3.1:1786172269.568485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fe00. 00010000:00000010:3.1:1786172269.568551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d800. 00010000:00000010:3.1:1786172269.602933:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392a00. 00010000:00000010:3.1:1786172269.642275:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541800. 00010000:00000010:3.1:1786172269.659413:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:3.1:1786172269.663529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e600. 00010000:00000010:3.1:1786172269.704409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541e00. 00010000:00000010:0.1:1786172269.708442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9c00. 00010000:00000010:0.1:1786172269.752178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:0.1:1786172269.752185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f000. 00010000:00000010:3.1:1786172269.774320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb800. 00010000:00000010:3.1:1786172269.774326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca600. 00010000:00000010:3.1:1786172269.784682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:3.1:1786172269.784690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:0.1:1786172269.869041:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9000. 00010000:00000010:3.1:1786172269.907393:0:178980:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:3.1:1786172269.944431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392800. 00010000:00000010:3.1:1786172269.969762:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8000. 00010000:00000010:0.1:1786172270.033783:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:0.1:1786172270.033796:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ee00. 00010000:00000010:0.1:1786172270.060230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:3.1:1786172270.060728:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:3.1:1786172270.077397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8c00. 00010000:00000010:3.1:1786172270.128705:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:0.1:1786172270.167408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d400. 00010000:00000010:0.1:1786172270.183304:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ce00. 00010000:00000010:3.1:1786172270.203379:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:0.1:1786172270.216101:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:3.1:1786172270.229873:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393400. 00010000:00000010:3.1:1786172270.235473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8c00. 00010000:00000010:3.1:1786172270.258874:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ca00. 00010000:00000010:3.1:1786172270.305773:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943400. 00010000:00000010:3.1:1786172270.305785:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9c00. 00010000:00000010:0.1:1786172270.305909:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8000. 00010000:00000010:3.1:1786172270.341639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cbc00. 00010000:00000010:3.1:1786172270.373067:0:152395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816e00. 00010000:00000010:3.1:1786172270.383399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540200. 00010000:00000010:3.1:1786172270.383410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541c00. 00010000:00000010:0.1:1786172270.402119:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817a00. 00010000:00000010:0.1:1786172270.402129:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:3.1:1786172270.434427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e800. 00010000:00000010:0.1:1786172270.470397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca600. 00010000:00000010:3.1:1786172270.503709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8c00. 00010000:00000010:3.1:1786172270.503723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610de00. 00010000:00000010:3.1:1786172270.567682:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e400. 00010000:00000010:3.1:1786172270.663133:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00010000:00000010:3.1:1786172270.685407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d600. 00010000:00000010:3.1:1786172270.685420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8200. 00010000:00000010:0.1:1786172270.708536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00010000:00000010:3.1:1786172270.712129:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ee00. 00010000:00000010:3.1:1786172270.712134:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817200. 00010000:00000010:3.1:1786172270.793399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540600. 00010000:00000010:0.1:1786172270.793513:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ec00. 00010000:00000010:3.1:1786172270.882105:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249c00. 00010000:00000010:3.1:1786172270.923800:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:3.1:1786172270.989925:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392e00. 00010000:00000010:0.1:1786172270.998981:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fe00. 00010000:00000010:3.1:1786172271.002416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817200. 00010000:00000010:3.1:1786172271.043542:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816e00. 00010000:00000010:3.1:1786172271.043551:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942400. 00010000:00000010:3.1:1786172271.043556:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393a00. 00010000:00000010:0.1:1786172271.043644:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817e00. 00010000:00000010:0.1:1786172271.043649:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ce00. 00010000:00000010:0.1:1786172271.043658:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb400. 00010000:00000010:0.1:1786172271.043660:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393600. 00010000:00000010:0.1:1786172271.051378:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817600. 00010000:00000010:0.1:1786172271.051386:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:3.1:1786172271.054603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3ca000. 00010000:00000010:3.1:1786172271.054609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540400. 00010000:00000010:3.1:1786172271.054614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942800. 00010000:00000010:3.1:1786172271.057617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123400. 00010000:00000010:0.1:1786172271.090387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:3.1:1786172271.137581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cb200. 00010000:00000010:0.1:1786172271.197848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f400. 00010000:00000010:3.1:1786172271.232377:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122de00. 00010000:00000010:3.1:1786172271.243849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:3.1:1786172271.243855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cba00. 00010000:00000010:3.1:1786172271.243858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393c00. 00010000:00000010:3.1:1786172271.243862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943e00. 00010000:00000010:3.1:1786172271.243864:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816400. 00010000:00000010:3.1:1786172271.243867:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e200. 00010000:00000010:3.1:1786172271.243870:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8600. 00010000:00000010:0.1:1786172271.253531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d000. 00010000:00000010:0.1:1786172271.289120:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e800. 00010000:00000010:3.1:1786172271.338742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:3.1:1786172271.341377:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ee00. 00010000:00000010:0.1:1786172271.358381:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc400. 00010000:00000010:3.1:1786172271.472564:0:152396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:3.1:1786172271.472572:0:152396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943200. 00010000:00000010:3.1:1786172271.472577:0:152396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:3.1:1786172271.472580:0:152396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:3.1:1786172271.472584:0:152396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39540e00. 00010000:00000010:3.1:1786172271.472588:0:152396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f200. 00010000:00000010:3.1:1786172271.472591:0:152396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d800. 00010000:00000010:0.1:1786172271.475521:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217c200. 00010000:00000010:0.1:1786172271.475530:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816000. 00010000:00000010:0.1:1786172271.475533:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943c00. 00010000:00000010:0.1:1786172271.475536:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8800. 00010000:00000010:0.1:1786172271.475539:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122200. 00010000:00000010:0.1:1786172271.475542:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:0.1:1786172271.480099:0:178992:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e200. 00010000:00000010:3.1:1786172271.482437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fcc00. 00010000:00000010:0.1:1786172271.489502:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:3.1:1786172271.588425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f200. 00010000:00000010:0.1:1786172271.625576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:3.1:1786172271.711200:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123200. 00010000:00000010:0.1:1786172271.777556:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:0.1:1786172271.777561:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:3.1:1786172271.800705:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610d600. 00010000:00000010:3.1:1786172271.822435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:0.1:1786172271.915457:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123800. 00010000:00000010:0.1:1786172271.939471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:3.1:1786172271.999405:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:0.1:1786172272.000271:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123c00. 00010000:00000010:0.1:1786172272.101391:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786172272.157633:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816600. 00010000:00000010:3.1:1786172272.185916:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fce00. 00010000:00000010:3.1:1786172272.210455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172272.256446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:0.1:1786172272.261477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:3.1:1786172272.368446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8a00. 00010000:00000010:0.1:1786172272.373705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:0.1:1786172272.440945:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:3.1:1786172272.447466:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:3.1:1786172272.447475:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:3.1:1786172272.447483:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e000. 00010000:00000010:3.1:1786172272.468469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd200. 00010000:00000010:0.1:1786172272.515379:0:1028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e400. 00010000:00000010:0.1:1786172272.518402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817600. 00010000:00000010:3.1:1786172272.520474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817200. 00010000:00000010:3.1:1786172272.524984:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817400. 00010000:00000010:3.1:1786172272.525001:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f200. 00010000:00000010:3.1:1786172272.598147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816c00. 00010000:00000010:0.1:1786172272.599930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f600. 00010000:00000010:3.1:1786172272.622731:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:3.1:1786172272.622737:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816a00. 00010000:00000010:0.1:1786172272.624504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:0.1:1786172272.629526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817200. 00010000:00000010:3.1:1786172272.674415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:0.1:1786172272.686496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172272.688474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543c00. 00010000:00000010:3.1:1786172272.791119:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:0.1:1786172272.866272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817400. 00010000:00000010:3.1:1786172272.870492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ee00. 00010000:00000010:3.1:1786172272.927390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc800. 00010000:00000010:3.1:1786172272.945558:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816e00. 00010000:00000010:0.1:1786172272.983503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816200. 00010000:00000010:3.1:1786172273.029591:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786172273.029839:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172273.071103:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217ce00. 00010000:00000010:0.1:1786172273.080531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:3.1:1786172273.093333:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817000. 00010000:00000010:0.1:1786172273.109608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:3.1:1786172273.183454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:3.1:1786172273.183462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:3.1:1786172273.206145:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543e00. 00010000:00000010:3.1:1786172273.206150:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817a00. 00010000:00000010:0.1:1786172273.267596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aaa00. 00010000:00000010:0.1:1786172273.274431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:0.1:1786172273.296395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:3.1:1786172273.340388:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa000. 00010000:00000010:3.1:1786172273.352771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817800. 00010000:00000010:3.1:1786172273.444165:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:0.1:1786172273.449389:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942a00. 00010000:00000010:0.1:1786172273.521416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542600. 00010000:00000010:0.1:1786172273.621695:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abc00. 00010000:00000010:3.1:1786172273.673544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e600. 00010000:00000010:3.1:1786172273.673565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543c00. 00010000:00000010:3.1:1786172273.673569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:3.1:1786172273.712398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543800. 00010000:00000010:0.1:1786172273.744375:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abc00. 00010000:00000010:0.1:1786172273.755083:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817000. 00010000:00000010:3.1:1786172273.954616:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123c00. 00010000:00000010:3.1:1786172273.954623:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816c00. 00010000:00000010:0.1:1786172273.989378:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974600. 00010000:00000010:3.1:1786172273.993524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f400. 00010000:00000010:0.1:1786172274.024725:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172274.306553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa000. 00010000:00000010:3.1:1786172274.321391:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f000. 00010000:00000010:3.1:1786172274.343387:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f200. 00010000:00000010:3.1:1786172274.349395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089fa00. 00010000:00000010:3.1:1786172274.371564:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2da00. 00010000:00000010:3.1:1786172274.442380:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:3.1:1786172274.442385:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d800. 00010000:00000010:0.1:1786172274.478968:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e200. 00010000:00000010:3.1:1786172274.561314:0:152395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:3.1:1786172274.612473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9200. 00010000:00000010:3.1:1786172274.612483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:3.1:1786172274.838436:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fc00. 00010000:00000010:0.1:1786172274.864689:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e600. 00010000:00000010:3.1:1786172275.005396:0:178973:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e000. 00010000:00000010:0.1:1786172275.084944:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f400. 00010000:00000010:3.1:1786172275.148544:0:178205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e400. 00010000:00000010:3.1:1786172275.148639:0:178205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786172275.182385:0:178967:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089fc00. 00010000:00000010:3.1:1786172275.185869:0:178967:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31817200. 00010000:00000010:0.1:1786172275.295387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f200. 00010000:00000010:0.1:1786172275.295394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2cc00. 00010000:00000010:3.1:1786172275.361427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:0.1:1786172275.387427:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2da00. 00010000:00000010:0.1:1786172275.470390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:3.1:1786172275.482549:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:3.1:1786172275.482559:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543800. 00010000:00000010:3.1:1786172275.482569:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:0.1:1786172275.482965:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543400. 00010000:00000010:3.1:1786172275.491190:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e400. 00010000:00000010:3.1:1786172275.491205:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:0.1:1786172275.491809:0:152396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd600. 00010000:00000010:0.1:1786172275.491821:0:152396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172275.503990:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:0.1:1786172275.505825:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d200. 00010000:00000010:0.1:1786172275.511392:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816800. 00010000:00000010:0.1:1786172275.511400:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d800. 00010000:00000010:0.1:1786172275.528381:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:3.1:1786172275.534339:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543c00. 00010000:00000010:3.1:1786172275.543382:0:178974:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abc00. 00010000:00000010:3.1:1786172275.648407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:3.1:1786172275.704481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089e800. 00010000:00000010:3.1:1786172275.704487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543600. 00010000:00000010:3.1:1786172275.753965:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123200. 00010000:00000010:0.1:1786172275.823542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:3.1:1786172275.824404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172275.922561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e3cac00. 00010000:00000010:3.1:1786172275.926433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ee00. 00010000:00000010:0.1:1786172275.990740:0:176869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:3.1:1786172276.021934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6fe00. 00010000:00000010:3.1:1786172276.046429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:3.1:1786172276.074466:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f200. 00010000:00000010:3.1:1786172276.095408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abc00. 00010000:00000010:3.1:1786172276.095417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172276.106513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:0.1:1786172276.285386:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:0.1:1786172276.285396:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f000. 00010000:00000010:0.1:1786172276.285399:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6f800. 00010000:00000010:0.1:1786172276.295648:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:3.1:1786172276.296455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aaa00. 00010000:00000010:3.1:1786172276.296464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123c00. 00010000:00000010:3.1:1786172276.306791:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541a00. 00010000:00000010:3.1:1786172276.398505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aaa00. 00010000:00000010:3.1:1786172276.421402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:3.1:1786172276.478963:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172276.505091:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:0.1:1786172276.507745:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975400. 00010000:00000010:3.1:1786172276.512344:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd600. 00010000:00000010:3.1:1786172276.561599:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:0.1:1786172276.686704:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa000. 00010000:00000010:0.1:1786172276.717873:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:0.1:1786172276.808553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc400. 00010000:00000010:3.1:1786172276.809089:0:178992:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:3.1:1786172276.990487:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172276.990494:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089ea00. 00010000:00000010:3.1:1786172277.038713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:3.1:1786172277.043217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fcc00. 00010000:00000010:3.1:1786172277.065444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542000. 00010000:00000010:3.1:1786172277.065465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786172277.129427:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:0.1:1786172277.168972:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122e00. 00010000:00000010:0.1:1786172277.172176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:3.1:1786172277.394400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:0.1:1786172277.455436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172277.491660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:0.1:1786172277.538505:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:0.1:1786172277.568421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:0.1:1786172277.568425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6e200. 00010000:00000010:3.1:1786172277.591692:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fce00. 00010000:00000010:0.1:1786172277.699411:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:0.1:1786172277.725734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa600. 00010000:00000010:3.1:1786172277.743565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172277.786182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:3.1:1786172277.794420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd400. 00010000:00000010:0.1:1786172277.834393:0:178971:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:3.1:1786172277.861201:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:3.1:1786172277.861214:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc800. 00010000:00000010:3.1:1786172278.024992:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:3.1:1786172278.056407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:3.1:1786172278.056416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543e00. 00010000:00000010:3.1:1786172278.056431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974600. 00010000:00000010:0.1:1786172278.057029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542a00. 00010000:00000010:0.1:1786172278.057034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:0.1:1786172278.153391:0:178972:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:0.1:1786172278.187447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172278.407401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:3.1:1786172278.489417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:3.1:1786172278.568998:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:3.1:1786172278.663573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:0.1:1786172278.847013:0:179006:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:0.1:1786172278.884919:0:178995:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786172278.884925:0:178995:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:3.1:1786172278.886005:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f6ea00. 00010000:00000010:3.1:1786172278.886012:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:3.1:1786172278.886016:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974a00. 00010000:00000010:3.1:1786172278.886019:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786172278.892773:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:0.1:1786172278.897614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089f800. 00010000:00000010:0.1:1786172278.897631:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc400. 00010000:00000010:3.1:1786172278.901416:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31816400. 00010000:00000010:3.1:1786172278.901427:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:3.1:1786172279.338014:0:178967:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:3.1:1786172279.583440:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:3.1:1786172280.814502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:3.1:1786172281.123989:0:178967:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:3.1:1786172285.210635:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e2610ca00. 00010000:00000010:3.1:1786172291.059440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d392000. 00010000:00000010:0.1:1786172291.061444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39541200. 00010000:00000010:0.1:1786172291.061451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d393e00. 00010000:00000010:0.1:1786172304.544449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:0.1:1786172304.559066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:3.1:1786172304.580034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123400. 00010000:00000010:3.1:1786172304.596223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:0.1:1786172304.683395:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172304.703479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:0.1:1786172304.732712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:3.1:1786172304.795399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:3.1:1786172304.829394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:0.1:1786172304.829451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122800. 00010000:00000010:3.1:1786172304.849936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:3.1:1786172304.849942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:0.1:1786172304.904173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:3.1:1786172304.915193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172304.964387:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:3.1:1786172304.972384:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:3.1:1786172304.991997:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172305.018407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974a00. 00010000:00000010:3.1:1786172305.108152:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:3.1:1786172305.108166:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:0.1:1786172305.146971:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:3.1:1786172305.248804:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:3.1:1786172305.255461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:3.1:1786172305.280539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:3.1:1786172305.286397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172305.299793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:3.1:1786172305.350526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:3.1:1786172305.667481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:3.1:1786172305.675901:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172305.707419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:3.1:1786172305.727627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:3.1:1786172305.860408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:3.1:1786172305.865218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:0.1:1786172305.959415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:0.1:1786172305.984481:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:0.1:1786172306.035629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:0.1:1786172306.035650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:3.1:1786172306.122541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172306.202783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:3.1:1786172306.202798:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:3.1:1786172306.219828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:3.1:1786172306.219843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:3.1:1786172306.348502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172306.348515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786172306.451649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172306.453402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:3.1:1786172306.509422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786172306.566896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122200. 00010000:00000010:0.1:1786172306.566909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:3.1:1786172306.582389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:3.1:1786172306.659401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:3.1:1786172306.659410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:3.1:1786172306.700268:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:3.1:1786172306.750073:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:3.1:1786172306.827510:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122200. 00010000:00000010:3.1:1786172306.872403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123400. 00010000:00000010:3.1:1786172306.964048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:3.1:1786172307.003395:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:3.1:1786172307.053430:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:3.1:1786172307.056469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:0.1:1786172307.061409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:3.1:1786172307.093051:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:3.1:1786172307.104921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:0.1:1786172307.134519:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786172307.134525:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172307.134529:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123800. 00010000:00000010:0.1:1786172307.148707:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123200. 00010000:00000010:3.1:1786172307.199827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249c00. 00010000:00000010:3.1:1786172307.199832:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:3.1:1786172307.276524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:3.1:1786172307.296527:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249c00. 00010000:00000010:3.1:1786172307.303347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:3.1:1786172307.356642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:0.1:1786172307.374416:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975400. 00010000:00000010:0.1:1786172307.393480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:3.1:1786172307.419409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:0.1:1786172307.433393:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:3.1:1786172307.572778:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:0.1:1786172307.623517:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:3.1:1786172307.630050:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172307.630060:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:3.1:1786172307.706514:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:3.1:1786172307.741109:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:3.1:1786172307.741122:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:3.1:1786172307.815398:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:0.1:1786172307.832669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:3.1:1786172307.933677:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:3.1:1786172308.001410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:0.1:1786172308.072498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:0.1:1786172308.072505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:0.1:1786172308.175721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123200. 00010000:00000010:0.1:1786172308.234405:0:1028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:3.1:1786172308.246391:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172308.287603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123800. 00010000:00000010:0.1:1786172308.288520:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:0.1:1786172308.372233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:3.1:1786172308.445567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:3.1:1786172308.445590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122800. 00010000:00000010:0.1:1786172308.447542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:0.1:1786172308.451549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:0.1:1786172308.576814:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172308.677684:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:3.1:1786172308.760398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:3.1:1786172308.772411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:3.1:1786172308.832395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:0.1:1786172308.910521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:0.1:1786172308.977903:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:0.1:1786172308.977911:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:0.1:1786172309.015427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172309.018463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:0.1:1786172309.069398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:3.1:1786172309.092691:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786172309.122174:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:3.1:1786172309.147468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa600. 00010000:00000010:0.1:1786172309.259375:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:0.1:1786172309.259380:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172309.309810:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974a00. 00010000:00000010:0.1:1786172309.444851:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:3.1:1786172309.459422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:3.1:1786172309.475561:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172309.538455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:3.1:1786172309.556733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:3.1:1786172309.568594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:3.1:1786172309.633603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:3.1:1786172309.633617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:3.1:1786172309.716048:0:169250:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:3.1:1786172309.716065:0:169250:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:3.1:1786172309.716069:0:169250:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172309.867730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:0.1:1786172309.877555:0:1348:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:3.1:1786172309.926605:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:3.1:1786172309.926612:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:3.1:1786172310.184399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172310.429436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:3.1:1786172310.564690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123400. 00010000:00000010:3.1:1786172310.678582:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:0.1:1786172310.767913:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:3.1:1786172310.828058:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:3.1:1786172310.828081:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:0.1:1786172310.883400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:3.1:1786172310.941886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:0.1:1786172310.966545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:0.1:1786172310.976414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:3.1:1786172311.069408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172311.069419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122800. 00010000:00000010:3.1:1786172311.078729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:3.1:1786172311.078738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:3.1:1786172311.222797:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:0.1:1786172311.263565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:3.1:1786172311.270404:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:0.1:1786172311.275476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:0.1:1786172311.283408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:3.1:1786172311.499469:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974a00. 00010000:00000010:3.1:1786172311.567469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:0.1:1786172311.668957:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:3.1:1786172311.809765:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:3.1:1786172311.809774:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249c00. 00010000:00000010:3.1:1786172311.852447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:3.1:1786172311.857430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:3.1:1786172311.864439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:3.1:1786172311.910062:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:0.1:1786172312.020474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:3.1:1786172312.038409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:3.1:1786172312.058792:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:3.1:1786172312.268690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:0.1:1786172312.417733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:3.1:1786172312.420709:0:178993:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172312.444413:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:3.1:1786172312.444419:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:0.1:1786172312.457426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123200. 00010000:00000010:3.1:1786172312.486415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:0.1:1786172312.536834:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:3.1:1786172312.546447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122e00. 00010000:00000010:0.1:1786172312.583853:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:3.1:1786172312.695046:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:0.1:1786172312.749084:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:3.1:1786172312.811409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:3.1:1786172312.873320:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172313.018391:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:3.1:1786172313.061228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786172313.078904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:0.1:1786172313.078920:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974a00. 00010000:00000010:3.1:1786172313.143928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:3.1:1786172313.188416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123c00. 00010000:00000010:3.1:1786172313.188432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172313.188436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172313.251842:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:3.1:1786172313.265099:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:3.1:1786172313.559665:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:3.1:1786172313.672464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:3.1:1786172313.708936:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:0.1:1786172313.713385:0:178966:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172313.928451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172313.964430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:0.1:1786172313.973563:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:0.1:1786172314.025494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122200. 00010000:00000010:0.1:1786172314.112441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:3.1:1786172314.236778:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172314.281134:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:3.1:1786172314.283063:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:3.1:1786172314.315236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:0.1:1786172314.367534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:3.1:1786172314.374721:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:3.1:1786172314.434412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:3.1:1786172314.758889:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:3.1:1786172314.782611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:3.1:1786172314.802519:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:0.1:1786172314.936459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172314.936520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172315.064308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:3.1:1786172315.156836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:3.1:1786172315.202543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abc00. 00010000:00000010:3.1:1786172315.234766:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa600. 00010000:00000010:3.1:1786172315.439409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172315.514512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:0.1:1786172315.547541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:0.1:1786172315.566404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249c00. 00010000:00000010:3.1:1786172315.580863:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:0.1:1786172315.613831:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:3.1:1786172315.700451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172315.700478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:3.1:1786172315.700481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975400. 00010000:00000010:3.1:1786172315.851170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:0.1:1786172315.867844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:3.1:1786172315.909796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:3.1:1786172316.337506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:0.1:1786172316.413469:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:0.1:1786172316.413479:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aaa00. 00010000:00000010:0.1:1786172316.413483:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:3.1:1786172316.445685:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:3.1:1786172316.445693:0:178976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:0.1:1786172316.446403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974600. 00010000:00000010:3.1:1786172316.507481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:3.1:1786172316.636419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aac00. 00010000:00000010:0.1:1786172316.716528:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:0.1:1786172316.942386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:0.1:1786172317.016759:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172317.104399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:3.1:1786172317.104411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:0.1:1786172317.146661:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122800. 00010000:00000010:0.1:1786172317.146666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:3.1:1786172317.174330:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122e00. 00010000:00000010:3.1:1786172317.216063:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:3.1:1786172317.240394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172317.350035:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:0.1:1786172317.370921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:0.1:1786172317.579426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786172317.579441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:3.1:1786172317.616559:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:3.1:1786172317.669482:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:0.1:1786172317.705342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172317.828398:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:3.1:1786172317.919796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:0.1:1786172317.968871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:3.1:1786172318.043094:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:3.1:1786172318.078672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172318.087200:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:3.1:1786172318.096388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123800. 00010000:00000010:3.1:1786172318.152970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:3.1:1786172318.197397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:0.1:1786172318.207653:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172318.211594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:3.1:1786172318.249681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:0.1:1786172318.370859:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172318.386065:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172318.403515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249c00. 00010000:00000010:3.1:1786172318.403521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:3.1:1786172318.448600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:3.1:1786172318.472456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172318.476434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172318.591146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:3.1:1786172318.631977:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:0.1:1786172318.730378:0:178978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172318.781396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:3.1:1786172318.811832:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172318.835615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:3.1:1786172318.835623:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:3.1:1786172318.948405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:3.1:1786172319.061442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:3.1:1786172319.061457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:3.1:1786172319.141607:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:3.1:1786172319.145283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122800. 00010000:00000010:3.1:1786172319.177240:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:0.1:1786172319.202193:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:3.1:1786172319.231488:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:3.1:1786172319.257543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:0.1:1786172319.395777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:0.1:1786172319.395784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:3.1:1786172319.406402:0:178979:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:3.1:1786172319.431476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122e00. 00010000:00000010:3.1:1786172319.525545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:0.1:1786172319.685099:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:3.1:1786172319.788878:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aaa00. 00010000:00000010:3.1:1786172319.791413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:0.1:1786172319.810195:0:178977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:3.1:1786172319.902419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974600. 00010000:00000010:3.1:1786172320.006738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:3.1:1786172320.079386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:0.1:1786172320.261257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:0.1:1786172320.261264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:0.1:1786172320.319955:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:3.1:1786172320.350073:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:3.1:1786172320.575888:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:0.1:1786172320.614001:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974600. 00010000:00000010:0.1:1786172320.614013:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:0.1:1786172320.662515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:3.1:1786172320.717454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172320.747078:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:3.1:1786172320.781403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:3.1:1786172320.894763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:3.1:1786172320.899394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:3.1:1786172321.092645:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:3.1:1786172321.107377:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:3.1:1786172321.319472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:3.1:1786172321.474464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:3.1:1786172321.485429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:3.1:1786172321.516501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:3.1:1786172321.596399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:3.1:1786172321.805664:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abc00. 00010000:00000010:0.1:1786172322.013060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:3.1:1786172322.318317:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:3.1:1786172322.371233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:0.1:1786172332.607898:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:0.1:1786172332.711143:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122800. 00000080:00000010:3.1:1786172332.716161:0:2395:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776620. 00010000:00000010:3.1:1786172332.780380:0:179024:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123400. 00000080:00000010:3.1:1786172332.784419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd620. 00010000:00000010:3.1:1786172332.784438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:3.1:1786172332.834690:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:3.1:1786172332.925622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249c00. 00000080:00000010:3.1:1786172332.948923:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed14a0. 00010000:00000010:0.1:1786172332.961818:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:3.1:1786172332.978469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:3.1:1786172333.042524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aaa00. 00000080:00000010:0.1:1786172333.047738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296480. 00010000:00000010:3.1:1786172333.086165:0:179024:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:3.1:1786172333.192337:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:0.1:1786172333.308132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:3.1:1786172333.409419:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249c00. 00010000:00000010:0.1:1786172333.424959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:3.1:1786172333.441392:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123c00. 00010000:00000010:3.1:1786172333.557404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:0.1:1786172333.557501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aac00. 00010000:00000010:0.1:1786172333.765588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:3.1:1786172333.870687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:3.1:1786172333.870702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123800. 00010000:00000010:3.1:1786172333.870711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123200. 00010000:00000010:0.1:1786172333.898854:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:3.1:1786172334.120235:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172334.120243:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:0.1:1786172334.260545:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:3.1:1786172334.287584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:0.1:1786172334.334265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:0.1:1786172334.334278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:0.1:1786172334.446413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:0.1:1786172334.446436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:0.1:1786172334.446447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:3.1:1786172334.493516:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122800. 00010000:00000010:0.1:1786172334.689655:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:3.1:1786172334.728557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:3.1:1786172334.740552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122e00. 00010000:00000010:3.1:1786172334.786995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:3.1:1786172334.926788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:3.1:1786172334.942868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172334.972330:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172334.984337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:0.1:1786172335.200764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:3.1:1786172335.265428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122200. 00010000:00000010:0.1:1786172335.288635:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248400. 00010000:00000010:3.1:1786172335.297802:0:178995:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc200. 00010000:00000010:3.1:1786172335.314941:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fce00. 00010000:00000010:0.1:1786172335.324400:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:0.1:1786172335.442380:0:179006:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc800. 00010000:00000010:3.1:1786172335.468312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:0.1:1786172335.555514:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8600. 00010000:00000010:3.1:1786172335.607515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172335.651409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:3.1:1786172335.667487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:3.1:1786172335.667498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc400. 00010000:00000010:0.1:1786172335.674495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd400. 00010000:00000010:0.1:1786172335.677930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172335.718505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:3.1:1786172335.718519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8800. 00010000:00000010:0.1:1786172335.774465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8a00. 00010000:00000010:3.1:1786172335.805405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8600. 00010000:00000010:3.1:1786172335.880606:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:0.1:1786172335.937416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:0.1:1786172336.138926:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:3.1:1786172336.139877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:3.1:1786172336.178658:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8c00. 00010000:00000010:3.1:1786172336.237660:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8000. 00010000:00000010:3.1:1786172336.388403:0:179022:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:0.1:1786172336.415505:0:178991:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aaa00. 00010000:00000010:0.1:1786172336.448238:0:179006:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00010000:00000010:3.1:1786172336.550414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:3.1:1786172336.587490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943800. 00010000:00000010:3.1:1786172336.618896:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:3.1:1786172336.621858:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943e00. 00010000:00000010:3.1:1786172336.621877:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:0.1:1786172336.633727:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d400. 00010000:00000010:3.1:1786172336.658182:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172336.658194:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:0.1:1786172336.665887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c000. 00010000:00000010:0.1:1786172336.665908:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:0.1:1786172336.681418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943c00. 00010000:00000010:3.1:1786172336.684411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aac00. 00010000:00000010:3.1:1786172336.684432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:3.1:1786172336.716528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172336.750649:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9c00. 00010000:00000010:0.1:1786172336.824390:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943c00. 00010000:00000010:3.1:1786172336.833023:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99600. 00010000:00000010:3.1:1786172336.978448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122800. 00010000:00000010:3.1:1786172336.978453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943800. 00010000:00000010:0.1:1786172337.009391:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:0.1:1786172337.009416:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943400. 00010000:00000010:3.1:1786172337.047540:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942800. 00010000:00000010:0.1:1786172337.261455:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:0.1:1786172337.261463:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8c00. 00010000:00000010:0.1:1786172337.365143:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99400. 00010000:00000010:3.1:1786172337.494659:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00010000:00000010:0.1:1786172337.499478:0:179022:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98200. 00010000:00000010:0.1:1786172337.539430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:0.1:1786172337.539435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:0.1:1786172337.539440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:0.1:1786172337.544581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb26a00. 00010000:00000010:0.1:1786172337.569195:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aac00. 00010000:00000010:0.1:1786172337.569200:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8a00. 00010000:00000010:3.1:1786172337.577708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:3.1:1786172337.619049:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943c00. 00010000:00000010:3.1:1786172337.630282:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74e00. 00010000:00000010:0.1:1786172337.630461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99200. 00010000:00000010:0.1:1786172337.718505:0:178994:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123000. 00010000:00000010:0.1:1786172337.746536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942400. 00010000:00000010:0.1:1786172337.768071:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb27400. 00010000:00000010:3.1:1786172337.841156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb27000. 00010000:00000010:3.1:1786172337.871764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:3.1:1786172337.885515:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:3.1:1786172337.910403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb27a00. 00010000:00000010:0.1:1786172337.990491:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00010000:00000010:0.1:1786172338.096412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:3.1:1786172338.120516:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb26600. 00010000:00000010:3.1:1786172338.143571:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75c00. 00010000:00000010:0.1:1786172338.272409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb27e00. 00010000:00000010:3.1:1786172338.466560:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aaa00. 00010000:00000010:0.1:1786172338.473803:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74000. 00010000:00000010:0.1:1786172338.473816:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75400. 00010000:00000010:3.1:1786172338.523011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75a00. 00010000:00000010:3.1:1786172338.548566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74800. 00010000:00000010:3.1:1786172338.673349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb24800. 00010000:00000010:3.1:1786172338.683432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa000. 00010000:00000010:0.1:1786172338.774458:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75200. 00010000:00000010:3.1:1786172338.788530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:3.1:1786172338.791239:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:3.1:1786172338.792714:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb27c00. 00010000:00000010:3.1:1786172338.878491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74c00. 00010000:00000010:3.1:1786172338.908553:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb24c00. 00010000:00000010:0.1:1786172338.989373:0:178980:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:0.1:1786172339.003100:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98200. 00010000:00000010:3.1:1786172339.102420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb26e00. 00010000:00000010:0.1:1786172339.107132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75800. 00010000:00000010:0.1:1786172339.145396:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb25c00. 00010000:00000010:3.1:1786172339.172856:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75a00. 00010000:00000010:3.1:1786172339.258463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:0.1:1786172339.269617:0:137051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75000. 00010000:00000010:0.1:1786172339.306966:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb26400. 00010000:00000010:3.1:1786172339.367650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75200. 00010000:00000010:0.1:1786172339.546622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74a00. 00010000:00000010:3.1:1786172339.547477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:3.1:1786172339.579455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb27000. 00010000:00000010:0.1:1786172339.641355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb20e00. 00010000:00000010:3.1:1786172339.682471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75800. 00010000:00000010:0.1:1786172339.703796:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb21c00. 00010000:00000010:3.1:1786172339.877453:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb24a00. 00010000:00000010:0.1:1786172339.960023:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:3.1:1786172339.985962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:3.1:1786172340.005744:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb21e00. 00010000:00000010:0.1:1786172340.027034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb27e00. 00010000:00000010:3.1:1786172340.101520:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74000. 00010000:00000010:3.1:1786172340.281443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75800. 00010000:00000010:0.1:1786172340.346560:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb21000. 00010000:00000010:3.1:1786172340.426835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aaa00. 00010000:00000010:0.1:1786172340.431460:0:179016:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb26c00. 00010000:00000010:3.1:1786172340.431824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99600. 00010000:00000010:3.1:1786172340.431830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942400. 00010000:00000010:0.1:1786172340.435504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb27800. 00010000:00000010:3.1:1786172340.436239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98c00. 00010000:00000010:0.1:1786172340.485411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aac00. 00010000:00000010:0.1:1786172340.485432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb20000. 00010000:00000010:0.1:1786172340.485478:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98800. 00010000:00000010:3.1:1786172340.493384:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:3.1:1786172340.493391:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75400. 00010000:00000010:3.1:1786172340.493393:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:0.1:1786172340.502542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb20800. 00010000:00000010:3.1:1786172340.502588:0:178994:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98a00. 00010000:00000010:3.1:1786172340.523662:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:0.1:1786172340.644796:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943800. 00010000:00000010:0.1:1786172340.655827:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8200. 00010000:00000010:3.1:1786172340.698445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:3.1:1786172340.731029:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9200. 00010000:00000010:0.1:1786172340.757489:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00010000:00000010:3.1:1786172340.770573:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9e00. 00010000:00000010:0.1:1786172340.824746:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75400. 00010000:00000010:3.1:1786172340.837003:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943000. 00010000:00000010:3.1:1786172340.837010:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00010000:00000010:3.1:1786172340.837017:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb21400. 00010000:00000010:0.1:1786172340.876031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:0.1:1786172340.876045:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d400. 00010000:00000010:3.1:1786172341.050525:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74800. 00010000:00000010:3.1:1786172341.050533:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:3.1:1786172341.219448:0:178994:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122200. 00010000:00000010:3.1:1786172341.226528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc800. 00010000:00000010:3.1:1786172341.293434:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123600. 00010000:00000010:3.1:1786172341.400392:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942800. 00010000:00000010:3.1:1786172341.424432:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74200. 00010000:00000010:3.1:1786172341.424448:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d800. 00010000:00000010:0.1:1786172341.464459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:0.1:1786172341.464467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:3.1:1786172341.517446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd200. 00010000:00000010:3.1:1786172341.603148:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc200. 00010000:00000010:0.1:1786172341.612963:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:0.1:1786172341.624448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249000. 00010000:00000010:3.1:1786172341.747137:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa000. 00010000:00000010:3.1:1786172341.794543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:3.1:1786172342.092032:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75e00. 00010000:00000010:3.1:1786172342.092045:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75400. 00010000:00000010:3.1:1786172342.202820:0:179021:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:0.1:1786172342.307562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:0.1:1786172342.307585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:0.1:1786172342.307593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172342.384408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74600. 00010000:00000010:3.1:1786172342.384416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786172342.389922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c000. 00010000:00000010:3.1:1786172342.426789:0:178982:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:3.1:1786172342.518453:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74400. 00010000:00000010:3.1:1786172342.586299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c600. 00010000:00000010:3.1:1786172342.586307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172342.706440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786172343.029804:0:179017:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb25a00. 00010000:00000010:0.1:1786172343.029812:0:179017:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:3.1:1786172343.031862:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:3.1:1786172343.046830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:0.1:1786172343.064135:0:178985:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786172343.070013:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99400. 00010000:00000010:0.1:1786172343.111489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542000. 00010000:00000010:3.1:1786172343.123386:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172343.228078:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943800. 00010000:00000010:3.1:1786172343.270033:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c400. 00010000:00000010:3.1:1786172343.278623:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786172343.469205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:0.1:1786172343.485090:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:0.1:1786172343.633777:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514400. 00010000:00000010:3.1:1786172343.677566:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:3.1:1786172343.708111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:3.1:1786172343.714681:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543000. 00010000:00000010:3.1:1786172343.932043:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e516400. 00010000:00000010:3.1:1786172343.932048:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb27a00. 00010000:00000010:3.1:1786172343.979036:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786172344.145717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e516a00. 00010000:00000010:3.1:1786172344.183605:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514200. 00010000:00000010:3.1:1786172344.306510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514400. 00010000:00000010:0.1:1786172344.396018:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74c00. 00010000:00000010:3.1:1786172344.498459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515e00. 00010000:00000010:0.1:1786172344.579559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:3.1:1786172344.626503:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e516800. 00010000:00000010:3.1:1786172344.626559:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d000. 00010000:00000010:3.1:1786172344.655542:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515400. 00010000:00000010:3.1:1786172344.682391:0:179017:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ea00. 00010000:00000010:0.1:1786172344.694984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942600. 00010000:00000010:3.1:1786172344.737655:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:3.1:1786172344.737666:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e600. 00010000:00000010:3.1:1786172344.779576:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515c00. 00010000:00000010:3.1:1786172344.820440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123800. 00010000:00000010:0.1:1786172344.911618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e517400. 00010000:00000010:3.1:1786172344.925537:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943600. 00010000:00000010:3.1:1786172344.974707:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e200. 00010000:00000010:3.1:1786172345.032466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943400. 00010000:00000010:0.1:1786172345.052518:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542400. 00010000:00000010:0.1:1786172345.114428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:3.1:1786172345.168478:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:3.1:1786172345.225449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249c00. 00010000:00000010:3.1:1786172345.295324:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542c00. 00010000:00000010:3.1:1786172345.307836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:0.1:1786172345.380231:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f400. 00010000:00000010:0.1:1786172345.523396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248200. 00010000:00000010:3.1:1786172345.674336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172345.674354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ee00. 00010000:00000010:3.1:1786172345.741380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:0.1:1786172345.757127:0:179021:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:3.1:1786172345.819656:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75a00. 00010000:00000010:3.1:1786172345.871473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f600. 00010000:00000010:0.1:1786172345.896391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:3.1:1786172345.897283:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202fc00. 00010000:00000010:0.1:1786172345.904880:0:179016:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:3.1:1786172345.916106:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e517e00. 00010000:00000010:3.1:1786172345.916116:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943600. 00010000:00000010:0.1:1786172345.996885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122400. 00010000:00000010:0.1:1786172345.996892:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:3.1:1786172346.045378:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123200. 00010000:00000010:3.1:1786172346.140574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942600. 00010000:00000010:3.1:1786172346.140583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e000. 00010000:00000010:0.1:1786172346.155780:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:3.1:1786172346.193427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:3.1:1786172346.420681:0:178205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:3.1:1786172346.420692:0:178205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:3.1:1786172346.433404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:0.1:1786172346.434803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249200. 00010000:00000010:3.1:1786172346.454668:0:178985:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:0.1:1786172346.465526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f800. 00010000:00000010:3.1:1786172346.469209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943000. 00010000:00000010:3.1:1786172346.469230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98200. 00010000:00000010:3.1:1786172346.535451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543000. 00010000:00000010:3.1:1786172346.561089:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172346.578663:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:3.1:1786172346.592391:0:179022:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c000. 00010000:00000010:3.1:1786172346.897632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:0.1:1786172346.902441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542400. 00010000:00000010:3.1:1786172347.016180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:0.1:1786172347.022250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:3.1:1786172347.207774:0:179023:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e800. 00010000:00000010:3.1:1786172347.207780:0:179023:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c000. 00010000:00000010:0.1:1786172347.230639:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:0.1:1786172347.263394:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd600. 00010000:00000010:3.1:1786172347.329444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:0.1:1786172347.329519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122200. 00010000:00000010:3.1:1786172347.436838:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ca00. 00010000:00000010:0.1:1786172347.440986:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786172347.443228:0:179006:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542c00. 00010000:00000010:3.1:1786172347.444339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122000. 00010000:00000010:0.1:1786172347.481118:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74a00. 00010000:00000010:0.1:1786172347.481131:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:3.1:1786172347.481834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:3.1:1786172347.500828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:3.1:1786172347.500841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:3.1:1786172347.522784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd200. 00010000:00000010:3.1:1786172347.771498:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d200. 00010000:00000010:0.1:1786172347.780439:0:179031:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:0.1:1786172347.800554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249e00. 00010000:00000010:0.1:1786172347.908430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d000. 00010000:00000010:3.1:1786172347.918344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:0.1:1786172348.010524:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:3.1:1786172348.160431:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248800. 00010000:00000010:0.1:1786172348.174410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543e00. 00010000:00000010:0.1:1786172348.192608:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd400. 00010000:00000010:3.1:1786172348.238490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c800. 00010000:00000010:0.1:1786172348.416881:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00010000:00000010:0.1:1786172348.416889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:3.1:1786172349.051900:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00010000:00000010:3.1:1786172349.207525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:3.1:1786172349.323287:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00010000:00000010:3.1:1786172349.335385:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fce00. 00010000:00000010:3.1:1786172349.402608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:0.1:1786172349.462207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c400. 00010000:00000010:3.1:1786172349.535498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:3.1:1786172349.550476:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542400. 00010000:00000010:3.1:1786172349.634305:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc800. 00010000:00000010:3.1:1786172349.642163:0:178991:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:3.1:1786172349.650375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248a00. 00010000:00000010:3.1:1786172349.708989:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942800. 00010000:00000010:0.1:1786172349.760118:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00010000:00000010:3.1:1786172349.929804:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123e00. 00010000:00000010:3.1:1786172349.987546:0:179021:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aba00. 00010000:00000010:3.1:1786172350.161687:0:48:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aac00. 00010000:00000010:0.1:1786172350.188080:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543e00. 00010000:00000010:3.1:1786172350.297285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786172350.471596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa200. 00010000:00000010:3.1:1786172350.482559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb20200. 00010000:00000010:3.1:1786172350.482582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:0.1:1786172350.582578:0:179022:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123c00. 00010000:00000010:3.1:1786172350.639036:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172350.639046:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa800. 00010000:00000010:3.1:1786172350.675248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543200. 00010000:00000010:3.1:1786172350.710763:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543c00. 00010000:00000010:3.1:1786172350.775389:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d400. 00010000:00000010:3.1:1786172350.775396:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ca00. 00010000:00000010:0.1:1786172350.778549:0:179006:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb21000. 00010000:00000010:0.1:1786172350.998476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00010000:00000010:3.1:1786172351.039542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:3.1:1786172351.107955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122a00. 00010000:00000010:0.1:1786172351.109473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:3.1:1786172351.117649:0:178980:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb21400. 00010000:00000010:3.1:1786172351.134420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fce00. 00010000:00000010:3.1:1786172351.225469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543c00. 00010000:00000010:0.1:1786172351.234392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122600. 00010000:00000010:3.1:1786172351.303575:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543600. 00010000:00000010:0.1:1786172351.315804:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:0.1:1786172351.315818:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2de00. 00010000:00000010:0.1:1786172351.315826:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c200. 00010000:00000010:3.1:1786172351.316891:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb21800. 00010000:00000010:3.1:1786172351.316903:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122de00. 00010000:00000010:3.1:1786172351.316906:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26122c00. 00010000:00000010:0.1:1786172351.376525:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc200. 00010000:00000010:3.1:1786172351.381443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd400. 00010000:00000010:0.1:1786172351.426040:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312abe00. 00010000:00000010:0.1:1786172351.524408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fcc00. 00010000:00000010:0.1:1786172351.572407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542600. 00010000:00000010:0.1:1786172351.632787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:3.1:1786172351.793567:0:178205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d200. 00010000:00000010:3.1:1786172351.815488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543c00. 00010000:00000010:0.1:1786172351.856466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb4e600. 00010000:00000010:3.1:1786172351.996498:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:3.1:1786172352.084815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ce00. 00010000:00000010:0.1:1786172352.093687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00010000:00000010:3.1:1786172352.155377:0:152395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2dc00. 00010000:00000010:0.1:1786172352.160469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cff000. 00010000:00000010:0.1:1786172352.160482:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d400. 00010000:00000010:3.1:1786172352.411487:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543600. 00010000:00000010:3.1:1786172352.430401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:0.1:1786172352.509492:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:0.1:1786172352.523625:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:0.1:1786172352.542401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542400. 00010000:00000010:3.1:1786172352.834666:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202fc00. 00010000:00000010:3.1:1786172352.834677:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d400. 00010000:00000010:0.1:1786172352.836214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26123a00. 00010000:00000010:0.1:1786172352.836221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75200. 00010000:00000010:3.1:1786172352.842978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:0.1:1786172352.880419:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75a00. 00010000:00000010:0.1:1786172352.880428:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543800. 00010000:00000010:0.1:1786172352.880440:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00010000:00000010:3.1:1786172352.903170:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:3.1:1786172352.948545:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:3.1:1786172352.948558:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:3.1:1786172352.962459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217c200. 00010000:00000010:0.1:1786172352.977411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:3.1:1786172352.977481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786172353.020412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:3.1:1786172353.110389:0:178992:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98c00. 00010000:00000010:0.1:1786172353.126648:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:3.1:1786172353.137522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514c00. 00010000:00000010:3.1:1786172353.146791:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10859a00. 00010000:00000010:3.1:1786172353.232715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10859600. 00010000:00000010:3.1:1786172353.256235:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514200. 00010000:00000010:3.1:1786172353.321404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99200. 00010000:00000010:3.1:1786172353.321424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10858200. 00010000:00000010:0.1:1786172353.324506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249a00. 00010000:00000010:0.1:1786172353.324512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:0.1:1786172353.395198:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e400. 00010000:00000010:0.1:1786172353.406816:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00010000:00000010:3.1:1786172353.437460:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943000. 00010000:00000010:3.1:1786172353.469735:0:178983:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514600. 00010000:00000010:3.1:1786172353.479942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f000. 00010000:00000010:3.1:1786172353.538438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00010000:00000010:0.1:1786172353.567617:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10859c00. 00010000:00000010:3.1:1786172353.612416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:0.1:1786172353.640580:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122de00. 00010000:00000010:3.1:1786172353.662405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31360200. 00010000:00000010:0.1:1786172353.681440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31360a00. 00010000:00000010:0.1:1786172353.764414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:0.1:1786172353.838414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:3.1:1786172353.864371:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ea00. 00010000:00000010:3.1:1786172353.864378:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31360600. 00010000:00000010:0.1:1786172353.906707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31361c00. 00010000:00000010:0.1:1786172353.906715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31361800. 00010000:00000010:3.1:1786172353.923582:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10859400. 00010000:00000010:3.1:1786172353.923596:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515200. 00010000:00000010:3.1:1786172353.985401:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:0.1:1786172353.996429:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:3.1:1786172354.164752:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e01000. 00010000:00000010:0.1:1786172354.192192:0:166014:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10859c00. 00010000:00000010:0.1:1786172354.196529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10858200. 00010000:00000010:0.1:1786172354.200386:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98800. 00010000:00000010:0.1:1786172354.215507:0:179017:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98200. 00010000:00000010:3.1:1786172354.286594:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10859800. 00010000:00000010:3.1:1786172354.333745:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:3.1:1786172354.333752:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:0.1:1786172354.334386:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd200. 00010000:00000010:0.1:1786172354.334395:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515600. 00010000:00000010:0.1:1786172354.350181:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249800. 00010000:00000010:0.1:1786172354.388600:0:179024:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e01a00. 00010000:00000010:0.1:1786172354.426384:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d200. 00010000:00000010:0.1:1786172354.444663:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d400. 00010000:00000010:3.1:1786172354.454037:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:3.1:1786172354.507103:0:179022:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:3.1:1786172354.507111:0:179022:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10858a00. 00010000:00000010:0.1:1786172354.512390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:3.1:1786172354.512439:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00010000:00000010:0.1:1786172354.519418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10858c00. 00010000:00000010:3.1:1786172354.563429:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fb21a00. 00010000:00000010:0.1:1786172354.647975:0:152386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e01000. 00010000:00000010:3.1:1786172354.666444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d200. 00010000:00000010:3.1:1786172354.666453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e00c00. 00010000:00000010:3.1:1786172354.680999:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00010000:00000010:3.1:1786172354.784013:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d400. 00010000:00000010:0.1:1786172354.858946:0:179031:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00010000:00000010:0.1:1786172354.975718:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f200. 00010000:00000010:0.1:1786172354.975729:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515800. 00010000:00000010:3.1:1786172355.017559:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:0.1:1786172355.099513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98200. 00010000:00000010:3.1:1786172355.129698:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:0.1:1786172355.224405:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98800. 00010000:00000010:3.1:1786172355.237093:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:3.1:1786172355.294885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514800. 00010000:00000010:0.1:1786172355.401386:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10859600. 00010000:00000010:3.1:1786172355.418935:0:179021:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:0.1:1786172355.590947:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:3.1:1786172355.627196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31360400. 00010000:00000010:3.1:1786172355.627201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfe000. 00010000:00000010:0.1:1786172355.651699:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:0.1:1786172355.840241:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e00c00. 00010000:00000010:3.1:1786172355.901111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515000. 00010000:00000010:0.1:1786172355.963416:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:0.1:1786172356.026173:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e200. 00010000:00000010:0.1:1786172356.090385:0:179020:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cff400. 00010000:00000010:3.1:1786172356.097461:0:178985:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00010000:00000010:3.1:1786172356.105527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab600. 00010000:00000010:3.1:1786172356.105536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786172356.105540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e01e00. 00010000:00000010:0.1:1786172356.213412:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10858000. 00010000:00000010:0.1:1786172356.213425:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:0.1:1786172356.213428:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10858e00. 00010000:00000010:0.1:1786172356.213431:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e00200. 00010000:00000010:0.1:1786172356.213473:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98c00. 00010000:00000010:0.1:1786172356.213478:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515200. 00010000:00000010:3.1:1786172356.229548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfe000. 00010000:00000010:0.1:1786172356.230411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542a00. 00010000:00000010:0.1:1786172356.468474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d800. 00010000:00000010:3.1:1786172356.558608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:3.1:1786172356.599833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfec00. 00010000:00000010:3.1:1786172356.599848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99800. 00010000:00000010:0.1:1786172356.608398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514800. 00010000:00000010:3.1:1786172356.608406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786172356.608409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98a00. 00010000:00000010:0.1:1786172356.612794:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74a00. 00010000:00000010:0.1:1786172356.668848:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e400. 00010000:00000010:3.1:1786172356.672416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cffa00. 00010000:00000010:3.1:1786172356.672430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99600. 00010000:00000010:3.1:1786172356.835671:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00010000:00000010:0.1:1786172357.040878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ca00. 00010000:00000010:3.1:1786172357.062815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542800. 00010000:00000010:3.1:1786172357.123032:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2da00. 00010000:00000010:3.1:1786172357.235208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc800. 00010000:00000010:3.1:1786172357.268393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542e00. 00010000:00000010:0.1:1786172357.301652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10859000. 00010000:00000010:0.1:1786172357.301657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74200. 00010000:00000010:3.1:1786172357.302532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cff400. 00010000:00000010:3.1:1786172357.302538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f000. 00010000:00000010:0.1:1786172357.305540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f400. 00010000:00000010:0.1:1786172357.427049:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98200. 00010000:00000010:3.1:1786172357.448355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99000. 00010000:00000010:3.1:1786172357.448387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98600. 00010000:00000010:3.1:1786172357.577679:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfe600. 00010000:00000010:3.1:1786172357.577684:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2cc00. 00010000:00000010:3.1:1786172357.613585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00010000:00000010:3.1:1786172357.637709:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cff400. 00010000:00000010:3.1:1786172357.778843:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786172357.778848:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:3.1:1786172357.780459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ce00. 00010000:00000010:3.1:1786172357.780464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75600. 00010000:00000010:3.1:1786172357.780466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99400. 00010000:00000010:3.1:1786172357.780469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd200. 00010000:00000010:3.1:1786172357.780478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00010000:00000010:3.1:1786172357.780484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ea00. 00010000:00000010:0.1:1786172357.780513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ca00. 00010000:00000010:0.1:1786172357.780522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10859400. 00010000:00000010:0.1:1786172357.780527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515800. 00010000:00000010:0.1:1786172357.780534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543e00. 00010000:00000010:0.1:1786172357.780537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10859200. 00010000:00000010:0.1:1786172357.780543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:0.1:1786172357.780547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c600. 00010000:00000010:0.1:1786172357.780550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e01a00. 00010000:00000010:0.1:1786172357.780554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202fc00. 00010000:00000010:0.1:1786172357.780557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e4217d400. 00010000:00000010:3.1:1786172358.009434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c600. 00010000:00000010:3.1:1786172358.052404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514e00. 00010000:00000010:3.1:1786172358.073454:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00010000:00000010:3.1:1786172358.221758:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514800. 00010000:00000010:3.1:1786172358.221763:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cff600. 00010000:00000010:0.1:1786172358.302393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248600. 00010000:00000010:3.1:1786172358.424681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514400. 00010000:00000010:3.1:1786172358.424687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515400. 00010000:00000010:3.1:1786172358.487814:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515a00. 00010000:00000010:0.1:1786172358.521710:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f600. 00010000:00000010:0.1:1786172358.570110:0:179006:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:0.1:1786172358.570124:0:179006:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786172358.573932:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9200. 00010000:00000010:0.1:1786172358.605241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515600. 00010000:00000010:3.1:1786172358.670164:0:178990:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8800. 00010000:00000010:3.1:1786172358.674210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514800. 00010000:00000010:0.1:1786172358.739165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c600. 00010000:00000010:0.1:1786172358.752719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00010000:00000010:0.1:1786172358.794389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172358.830723:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:3.1:1786172358.932342:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:0.1:1786172358.940929:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00010000:00000010:0.1:1786172358.995488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:0.1:1786172358.995493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ce00. 00010000:00000010:3.1:1786172359.006122:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8600. 00010000:00000010:3.1:1786172359.013140:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:0.1:1786172359.062454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00010000:00000010:0.1:1786172359.088390:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98200. 00010000:00000010:3.1:1786172359.105375:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99600. 00010000:00000010:3.1:1786172359.219562:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c000. 00010000:00000010:3.1:1786172359.257659:0:179024:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00010000:00000010:3.1:1786172359.257671:0:179024:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248e00. 00010000:00000010:0.1:1786172359.277415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00010000:00000010:0.1:1786172359.311400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98c00. 00010000:00000010:0.1:1786172359.337568:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122de00. 00010000:00000010:3.1:1786172359.389023:0:178983:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab800. 00010000:00000010:0.1:1786172359.407856:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249400. 00010000:00000010:3.1:1786172359.416036:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98800. 00010000:00000010:3.1:1786172359.474882:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d000. 00010000:00000010:3.1:1786172359.527622:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00010000:00000010:0.1:1786172359.533497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:3.1:1786172359.538235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974600. 00010000:00000010:0.1:1786172359.607479:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99800. 00010000:00000010:0.1:1786172359.607493:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98800. 00010000:00000010:0.1:1786172359.616576:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99200. 00010000:00000010:3.1:1786172359.644432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00010000:00000010:0.1:1786172359.699513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98a00. 00010000:00000010:3.1:1786172359.739053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98c00. 00010000:00000010:0.1:1786172359.810383:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab000. 00010000:00000010:3.1:1786172359.829111:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248c00. 00010000:00000010:3.1:1786172359.837079:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:3.1:1786172359.837084:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ce00. 00010000:00000010:3.1:1786172359.895435:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2de00. 00010000:00000010:3.1:1786172359.993334:0:178983:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10248000. 00010000:00000010:3.1:1786172359.993353:0:178983:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00010000:00000010:3.1:1786172360.007693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:0.1:1786172360.007753:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99000. 00010000:00000010:3.1:1786172360.018284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2ce00. 00010000:00000010:0.1:1786172360.044289:0:178983:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10249600. 00010000:00000010:3.1:1786172360.047146:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d000. 00010000:00000010:0.1:1786172360.058727:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:0.1:1786172360.063420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cff000. 00010000:00000010:3.1:1786172360.207650:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00010000:00000010:0.1:1786172360.259518:0:179023:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd600. 00010000:00000010:0.1:1786172360.264614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab400. 00010000:00000010:0.1:1786172360.322408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00010000:00000010:3.1:1786172360.465515:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514e00. 00010000:00000010:0.1:1786172360.477088:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00010000:00000010:0.1:1786172360.617843:0:179028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a8800. 00010000:00000010:3.1:1786172360.635425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00010000:00000010:3.1:1786172360.675120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:3.1:1786172360.692897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:0.1:1786172360.725559:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542200. 00010000:00000010:3.1:1786172360.795309:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfea00. 00010000:00000010:0.1:1786172360.815489:0:179016:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:0.1:1786172360.859483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:0.1:1786172360.863844:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:3.1:1786172360.920843:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98600. 00010000:00000010:3.1:1786172360.970090:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfe800. 00010000:00000010:0.1:1786172361.006400:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c000. 00010000:00000010:3.1:1786172361.102077:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:3.1:1786172361.146131:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfee00. 00010000:00000010:0.1:1786172361.181228:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfe000. 00010000:00000010:0.1:1786172361.208576:0:178992:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75400. 00010000:00000010:0.1:1786172361.215432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ea00. 00010000:00000010:3.1:1786172361.255413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:3.1:1786172361.281679:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543400. 00010000:00000010:3.1:1786172361.349784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfea00. 00010000:00000010:0.1:1786172361.351400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543200. 00010000:00000010:3.1:1786172361.477942:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00010000:00000010:0.1:1786172361.496411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542a00. 00010000:00000010:3.1:1786172361.686686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31360400. 00010000:00000010:0.1:1786172361.711441:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75c00. 00010000:00000010:0.1:1786172361.725405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aae00. 00010000:00000010:0.1:1786172361.725419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ee00. 00010000:00000010:0.1:1786172361.731556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542000. 00010000:00000010:3.1:1786172361.731599:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f000. 00010000:00000010:3.1:1786172361.735341:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfe400. 00010000:00000010:0.1:1786172361.835965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514200. 00010000:00000010:3.1:1786172361.837098:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74c00. 00010000:00000010:0.1:1786172361.869571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202fe00. 00010000:00000010:3.1:1786172361.890393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa600. 00010000:00000010:3.1:1786172361.952371:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99800. 00010000:00000010:0.1:1786172361.981427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5000. 00010000:00000010:3.1:1786172361.985462:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cff600. 00010000:00000010:3.1:1786172362.003422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfea00. 00010000:00000010:0.1:1786172362.009964:0:152385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5800. 00010000:00000010:0.1:1786172362.021378:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfec00. 00010000:00000010:3.1:1786172362.075070:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75800. 00010000:00000010:3.1:1786172362.191037:0:179024:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4a00. 00010000:00000010:0.1:1786172362.215556:0:179006:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5400. 00010000:00000010:0.1:1786172362.219060:0:152395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31361800. 00010000:00000010:3.1:1786172362.222379:0:178991:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cfe200. 00010000:00000010:0.1:1786172362.231378:0:166014:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31361e00. 00010000:00000010:0.1:1786172362.231393:0:166014:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31361600. 00010000:00000010:3.1:1786172362.241450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4000. 00010000:00000010:0.1:1786172362.242420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5600. 00010000:00000010:0.1:1786172362.332061:0:179027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237000. 00010000:00000010:3.1:1786172362.332375:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f75e00. 00010000:00000010:3.1:1786172362.332380:0:179025:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f74800. 00010000:00000010:3.1:1786172362.369560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236200. 00010000:00000010:0.1:1786172362.461520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4e00. 00010000:00000010:0.1:1786172362.461542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31361600. 00010000:00000010:0.1:1786172362.461552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31360800. 00010000:00000010:0.1:1786172362.461555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99800. 00010000:00000010:3.1:1786172362.486696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236a00. 00010000:00000010:0.1:1786172362.554852:0:179017:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31361a00. 00010000:00000010:3.1:1786172362.593835:0:178982:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31361800. 00010000:00000010:3.1:1786172362.617875:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d400. 00010000:00000010:0.1:1786172362.790962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31360c00. 00010000:00000010:0.1:1786172362.790967:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236e00. 00010000:00000010:3.1:1786172362.869030:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4600. 00010000:00000010:0.1:1786172362.966839:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31361800. 00010000:00000010:0.1:1786172362.983389:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99000. 00010000:00000010:0.1:1786172362.996766:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31360000. 00010000:00000010:0.1:1786172363.072415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5600. 00010000:00000010:0.1:1786172363.079524:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98600. 00010000:00000010:3.1:1786172363.082395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4000. 00010000:00000010:0.1:1786172363.160510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236400. 00010000:00000010:3.1:1786172363.191414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514000. 00010000:00000010:3.1:1786172363.233862:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236e00. 00010000:00000010:0.1:1786172363.256031:0:179026:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5c00. 00010000:00000010:0.1:1786172363.399455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237200. 00010000:00000010:3.1:1786172363.486634:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236c00. 00010000:00000010:3.1:1786172363.551535:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5600. 00010000:00000010:3.1:1786172363.614092:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4400. 00010000:00000010:3.1:1786172363.856401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236e00. 00010000:00000010:3.1:1786172363.856409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31361200. 00010000:00000010:3.1:1786172364.052378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31360e00. 00010000:00000010:0.1:1786172364.129473:0:179022:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e515e00. 00010000:00000010:3.1:1786172364.397389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4e00. 00010000:00000010:3.1:1786172364.440919:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:3.1:1786172364.856393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514a00. 00010000:00000010:3.1:1786172364.978098:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236800. 00010000:00000010:3.1:1786172365.257263:0:1028:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943400. 00010000:00000010:3.1:1786172365.999498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cff000. 00010000:00000010:3.1:1786172366.158384:0:179023:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5800. 00010000:00000010:3.1:1786172366.194396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312aa400. 00010000:00000010:3.1:1786172366.914417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fce00. 00010000:00000010:0.1:1786172367.153852:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236400. 00010000:00000010:0.1:1786172367.153859:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236e00. 00010000:00000010:3.1:1786172367.215918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00010000:00000010:3.1:1786172367.580708:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4c00. 00010000:00000010:3.1:1786172368.117433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4200. 00010000:00000010:3.1:1786172368.224166:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459800. 00010000:00000010:3.1:1786172368.224174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:3.1:1786172368.236383:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786172368.257650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39542600. 00010000:00000010:3.1:1786172368.314478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99800. 00010000:00000010:3.1:1786172368.651353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9e00. 00010000:00000010:3.1:1786172368.651393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e11cff200. 00010000:00000010:3.1:1786172368.651403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2dc00. 00010000:00000010:3.1:1786172368.767107:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459b00. 00010000:00000010:3.1:1786172368.767120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:0.1:1786172369.197474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ea00. 00010000:00000010:3.1:1786172369.230648:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3100. 00010000:00000010:3.1:1786172369.230665:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f000. 00010000:00000010:0.1:1786172370.333398:0:179016:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401900. 00010000:00000010:0.1:1786172370.333408:0:179016:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236c00. 00010000:00000010:0.1:1786172370.333411:0:179016:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236200. 00010000:00000010:0.1:1786172370.443442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e200. 00010000:00000010:3.1:1786172370.447631:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e000. 00010000:00000010:3.1:1786172370.447638:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3300. 00010000:00000010:3.1:1786172370.447643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99200. 00010000:00000010:0.1:1786172370.474463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202fe00. 00010000:00000010:3.1:1786172370.487376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:3.1:1786172370.553790:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e000. 00010000:00000010:3.1:1786172370.553810:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3b00. 00010000:00000010:3.1:1786172370.553818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:3.1:1786172370.558455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa000. 00010000:00000010:3.1:1786172370.558465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99200. 00010000:00000010:3.1:1786172370.558468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00010000:00000010:3.1:1786172370.584733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98200. 00010000:00000010:3.1:1786172370.683567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:0.1:1786172370.685190:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e600. 00010000:00000010:0.1:1786172370.689518:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:0.1:1786172370.734079:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202fc00. 00010000:00000010:3.1:1786172370.756411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786172370.781575:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:3.1:1786172370.781584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3f00. 00010000:00000010:3.1:1786172370.781589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f600. 00010000:00000010:3.1:1786172370.832088:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00010000:00000010:0.1:1786172370.840677:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ec00. 00010000:00000010:0.1:1786172370.874398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f600. 00010000:00000010:0.1:1786172370.886404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236000. 00010000:00000010:0.1:1786172370.906882:0:179024:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459000. 00010000:00000010:0.1:1786172370.906889:0:179024:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236600. 00010000:00000010:0.1:1786172370.928284:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e800. 00010000:00000010:0.1:1786172370.934958:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236a00. 00010000:00000010:3.1:1786172371.003408:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3400. 00010000:00000010:3.1:1786172371.003421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ee00. 00010000:00000010:0.1:1786172371.045498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237200. 00010000:00000010:3.1:1786172371.177973:0:2395:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa400. 00010000:00000010:3.1:1786172371.177981:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237600. 00010000:00000010:3.1:1786172371.227082:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237200. 00010000:00000010:3.1:1786172371.323403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786172371.409432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:0.1:1786172371.567377:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:3.1:1786172371.618400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459500. 00010000:00000010:3.1:1786172371.618408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:3.1:1786172371.618411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:3.1:1786172371.731268:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00010000:00000010:3.1:1786172371.734438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa400. 00010000:00000010:3.1:1786172371.734446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98200. 00010000:00000010:0.1:1786172371.811559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f800. 00010000:00000010:3.1:1786172371.816968:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ee00. 00010000:00000010:3.1:1786172371.816974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:3.1:1786172371.878447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98200. 00010000:00000010:3.1:1786172371.878463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa400. 00010000:00000010:3.1:1786172371.878467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f800. 00010000:00000010:3.1:1786172371.990899:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaf00. 00010000:00000010:3.1:1786172371.990907:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98c00. 00010000:00000010:3.1:1786172371.990911:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237600. 00010000:00000010:3.1:1786172371.990914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99600. 00010000:00000010:0.1:1786172372.008879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237a00. 00010000:00000010:0.1:1786172372.008889:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401b00. 00010000:00000010:0.1:1786172372.008893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236400. 00010000:00000010:0.1:1786172372.092118:0:2386:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401600. 00010000:00000010:0.1:1786172372.092128:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237800. 00010000:00000010:0.1:1786172372.092132:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237c00. 00010000:00000010:3.1:1786172372.099402:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459c00. 00010000:00000010:3.1:1786172372.099414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:3.1:1786172372.130403:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401000. 00010000:00000010:3.1:1786172372.130412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:3.1:1786172372.162419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786172372.162426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236800. 00010000:00000010:3.1:1786172372.189379:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786172372.230442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:3.1:1786172372.230457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401a00. 00010000:00000010:3.1:1786172372.230461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236200. 00010000:00000010:0.1:1786172372.321235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e600. 00010000:00000010:0.1:1786172372.383254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236c00. 00010000:00000010:3.1:1786172372.388038:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401300. 00010000:00000010:3.1:1786172372.388047:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237600. 00010000:00000010:0.1:1786172372.439851:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f400. 00010000:00000010:3.1:1786172372.499579:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ee00. 00010000:00000010:3.1:1786172372.516253:0:179023:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:3.1:1786172372.706483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99800. 00010000:00000010:3.1:1786172372.742426:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa300. 00010000:00000010:3.1:1786172372.742433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98c00. 00010000:00000010:3.1:1786172372.788401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459900. 00010000:00000010:3.1:1786172372.788418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:3.1:1786172372.854607:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3e00. 00010000:00000010:3.1:1786172372.854618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e000. 00010000:00000010:3.1:1786172372.879421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99800. 00010000:00000010:0.1:1786172372.944977:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99800. 00010000:00000010:3.1:1786172372.954444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202fe00. 00010000:00000010:3.1:1786172372.954458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3600. 00010000:00000010:3.1:1786172372.954467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f400. 00010000:00000010:0.1:1786172373.015785:0:2393:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:0.1:1786172373.029755:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459400. 00010000:00000010:0.1:1786172373.029771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e800. 00010000:00000010:3.1:1786172373.171400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459400. 00010000:00000010:3.1:1786172373.171416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:0.1:1786172373.303745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202fc00. 00010000:00000010:3.1:1786172373.310404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00010000:00000010:0.1:1786172373.322473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e800. 00010000:00000010:0.1:1786172373.330447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3300. 00010000:00000010:0.1:1786172373.330455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e400. 00010000:00000010:3.1:1786172373.353821:0:179024:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:0.1:1786172373.368354:0:2385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459900. 00010000:00000010:0.1:1786172373.368554:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e800. 00010000:00000010:0.1:1786172373.494409:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:0.1:1786172373.494420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237800. 00010000:00000010:3.1:1786172373.725578:0:152396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:0.1:1786172373.728160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172373.769786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237a00. 00010000:00000010:0.1:1786172373.822285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f200. 00010000:00000010:0.1:1786172373.822292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e000. 00010000:00000010:0.1:1786172373.910270:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e800. 00010000:00000010:0.1:1786172373.910276:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f000. 00010000:00000010:3.1:1786172373.910975:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3400. 00010000:00000010:3.1:1786172373.910984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:0.1:1786172374.034106:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3c00. 00010000:00000010:0.1:1786172374.034114:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e600. 00010000:00000010:3.1:1786172374.061935:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401700. 00010000:00000010:3.1:1786172374.061949:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236a00. 00010000:00000010:0.1:1786172374.074674:0:179031:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237800. 00010000:00000010:3.1:1786172374.227573:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401700. 00010000:00000010:3.1:1786172374.227584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237600. 00010000:00000010:3.1:1786172374.472449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:0.1:1786172374.509379:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3c00. 00010000:00000010:0.1:1786172374.509390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202fa00. 00010000:00000010:3.1:1786172374.650490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f400. 00010000:00000010:3.1:1786172375.014044:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f800. 00010000:00000010:3.1:1786172375.069517:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa100. 00010000:00000010:3.1:1786172375.069533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:3.1:1786172375.230414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99c00. 00010000:00000010:3.1:1786172375.280157:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:0.1:1786172375.529505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172375.573462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:3.1:1786172375.850713:0:145910:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa100. 00010000:00000010:3.1:1786172375.850728:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:3.1:1786172375.876432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98800. 00010000:00000010:0.1:1786172375.945635:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3c00. 00010000:00000010:0.1:1786172375.945644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f200. 00010000:00000010:3.1:1786172376.004159:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459a00. 00010000:00000010:3.1:1786172376.004166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:3.1:1786172376.307688:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459c00. 00010000:00000010:3.1:1786172376.307699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:0.1:1786172376.328531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401300. 00010000:00000010:0.1:1786172376.328545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237400. 00010000:00000010:3.1:1786172376.359224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00000080:00000010:0.1:1786172376.392535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e092961c0. 00010000:00000010:3.1:1786172385.528411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99400. 00010000:00000010:0.1:1786172385.554051:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09237c00. 00010000:00000010:0.1:1786172385.557057:0:179043:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202e200. 00010000:00000010:0.1:1786172385.557069:0:179043:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00000080:00000010:3.1:1786172385.591388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1b40. 00000080:00000010:0.1:1786172385.610256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd960. 00000080:00000010:0.1:1786172385.610276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296880. 00010000:00000010:3.1:1786172385.751354:0:179049:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e514600. 00010000:00000010:3.1:1786172386.243003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202f800. 00010000:00000010:3.1:1786172386.489045:0:179046:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e09236a00. 00010000:00000010:0.1:1786172386.652379:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3202ec00. 00010000:00000010:3.1:1786172386.760043:0:179047:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00010000:00000010:0.1:1786172387.787003:0:179044:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786172387.787031:0:179044:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99400. 00010000:00000010:0.1:1786172388.166842:0:179043:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:0.1:1786172388.389508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:3.1:1786172388.693427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c400. 00010000:00000010:3.1:1786172390.157963:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:0.1:1786172390.163976:0:179046:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2cc00. 00010000:00000010:0.1:1786172390.748406:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459000. 00010000:00000010:0.1:1786172390.748419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00000080:00000010:0.1:1786172400.765903:0:2393:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbd600. 00010000:00000010:0.1:1786172400.795022:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:0.1:1786172400.845020:0:179031:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e014a9c00. 00000080:00000010:3.1:1786172400.886172:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02fbdf60. 00010000:00000010:3.1:1786172400.933878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172400.949863:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00000080:00000010:3.1:1786172401.014170:0:179051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296960. 00000080:00000010:3.1:1786172401.071606:0:179062:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776620. 00000080:00000010:3.1:1786172401.157001:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776ae0. 00010000:00000010:3.1:1786172401.582417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e06023200. 00010000:00000010:3.1:1786172401.602673:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3a00. 00010000:00000010:3.1:1786172401.602681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:3.1:1786172402.363268:0:145910:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401a00. 00010000:00000010:3.1:1786172402.363278:0:145910:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2c800. 00010000:00000010:0.1:1786172403.080480:0:179063:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401000. 00010000:00000010:0.1:1786172403.080487:0:179063:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543200. 00010000:00000010:3.1:1786172404.447472:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa300. 00010000:00000010:3.1:1786172404.447478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00000080:00000010:0.1:1786172404.494504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296380. 00010000:00000010:0.1:1786172407.010063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786172407.010080:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e312ab200. 00010000:00000010:0.1:1786172407.010085:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974000. 00010000:00000010:0.1:1786172407.010090:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459700. 00010000:00000010:0.1:1786172407.010093:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31f2d600. 00010000:00000010:0.1:1786172407.010097:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3500. 00010000:00000010:0.1:1786172407.010101:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e39543a00. 00010000:00000010:3.1:1786172505.643949:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aae00. 00010000:00000010:3.1:1786172505.643959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00010000:00000010:3.1:1786172512.229473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa600. 00010000:00000010:3.1:1786172512.229492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00010000:00000010:3.1:1786172512.257416:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459300. 00010000:00000010:3.1:1786172512.257439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98c00. 00010000:00000010:0.1:1786172512.272414:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459200. 00010000:00000010:0.1:1786172512.272434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172512.381123:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459800. 00010000:00000010:3.1:1786172512.381138:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786172512.466416:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459000. 00000800:00000200:3.1:1786172513.738978:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 38484 low water 2077910 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.739151:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 38484 low water 2077910 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.739419:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 37204 low water 2078550 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.739882:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 37204 low water 2078550 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.740181:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 35924 low water 2079190 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.740583:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 34644 low water 2079830 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.740595:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 34644 low water 2079830 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.740985:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 145860 low water 2024222 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.741287:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 145860 low water 2024222 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.741936:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 145860 low water 2024222 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.742243:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 145860 low water 2024222 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.742280:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 144580 low water 2024862 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.742289:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 144580 low water 2024862 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.743985:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 441156 low water 1876574 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.744008:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 441156 low water 1876574 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.744021:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 441156 low water 1876574 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.744034:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 441156 low water 1876574 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.744040:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 441156 low water 1876574 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.745597:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 441156 low water 1876574 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.745617:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 441156 low water 1876574 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.745628:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 441156 low water 1876574 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.746372:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 441156 low water 1876574 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.746395:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 441156 low water 1876574 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.746417:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 439876 low water 1877214 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.750508:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 869392 low water 1662456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.750529:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 869392 low water 1662456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.750539:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 869392 low water 1662456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.750552:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 869392 low water 1662456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.750579:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 869392 low water 1662456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.750587:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 869392 low water 1662456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.751172:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 869392 low water 1662456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.751188:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 869392 low water 1662456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.751201:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 869392 low water 1662456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.751212:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 869392 low water 1662456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.751223:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 869392 low water 1662456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.752337:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1025348 low water 1584478 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.752355:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1025348 low water 1584478 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.752365:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1025348 low water 1584478 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.752374:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1025348 low water 1584478 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.752386:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1025348 low water 1584478 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.752391:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1025348 low water 1584478 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.753020:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1025348 low water 1584478 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.753038:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1025348 low water 1584478 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.753049:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1025348 low water 1584478 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.753068:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1024068 low water 1585118 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.753621:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1061140 low water 1566582 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.753627:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1061140 low water 1566582 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172513.756609:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1636384 low water 1278960 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.038361:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.038383:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.038398:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.038408:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.038416:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.039825:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487592 low water 1853356 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.039841:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487592 low water 1853356 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.039849:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487592 low water 1853356 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.039858:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487592 low water 1853356 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.039870:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487592 low water 1853356 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.039875:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487592 low water 1853356 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.041335:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.041369:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.041382:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.041394:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.041404:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.041410:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.045938:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.045965:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.045978:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.045989:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.046003:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.048201:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.048215:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.048226:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.048235:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.048243:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.048491:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172515.048529:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1509832 low water 1342236 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.135488:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 103400 low water 2045452 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.135512:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 102120 low water 2046092 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.135519:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 102120 low water 2046092 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.141299:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 102120 low water 2046092 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.141592:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 100840 low water 2046732 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.141772:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 99560 low water 2047372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.142009:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 99560 low water 2047372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.142566:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 98280 low water 2048012 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.142595:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 97000 low water 2048652 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.142606:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 97000 low water 2048652 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.143812:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 123844 low water 2035230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.144161:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 122564 low water 2035870 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.144499:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 122564 low water 2035870 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.144506:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 122564 low water 2035870 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.144572:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 159636 low water 2017334 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.144586:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 159636 low water 2017334 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.144593:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 158356 low water 2017974 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.144624:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 252956 low water 1970674 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.144635:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 252956 low water 1970674 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.144640:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 252956 low water 1970674 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.144656:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 291308 low water 1951498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.149456:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 357784 low water 1918260 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.149499:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 357784 low water 1918260 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.149505:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 357784 low water 1918260 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.149566:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 488176 low water 1853064 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.149572:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 488176 low water 1853064 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.152687:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679924 low water 1757190 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.152708:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679924 low water 1757190 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.152720:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679924 low water 1757190 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.152725:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679924 low water 1757190 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.156335:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679924 low water 1757190 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.156379:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679924 low water 1757190 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.156392:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679924 low water 1757190 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.157257:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.157274:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.157285:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.157295:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.157300:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.159453:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.159469:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.159479:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.159489:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.160591:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.163740:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1319084 low water 1437610 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.163756:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1319084 low water 1437610 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.163767:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1319084 low water 1437610 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.163776:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1319084 low water 1437610 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.163787:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1319084 low water 1437610 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.163792:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1319084 low water 1437610 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172516.163823:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1574748 low water 1309778 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.959935:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 19028 low water 2087638 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.959946:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 19028 low water 2087638 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.960399:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 17748 low water 2088278 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.966575:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 140472 low water 2026916 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.966614:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 139192 low water 2027556 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.966625:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 139192 low water 2027556 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.967953:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 213336 low water 1990484 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.967970:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 212056 low water 1991124 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.967975:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 212056 low water 1991124 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.973157:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 212056 low water 1991124 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.975121:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 210776 low water 1991764 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.975470:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 210776 low water 1991764 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.975513:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 209496 low water 1992404 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.975528:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 209496 low water 1992404 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.976341:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 282360 low water 1955972 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.976456:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 282360 low water 1955972 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.976472:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 282360 low water 1955972 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.976478:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 282360 low water 1955972 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.976508:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 347556 low water 1923374 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.977337:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 422980 low water 1885662 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.977547:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 422980 low water 1885662 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.977771:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 422980 low water 1885662 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.977794:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 422980 low water 1885662 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.977806:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 422980 low water 1885662 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.977811:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 422980 low water 1885662 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.982474:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743840 low water 1725232 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.982672:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743840 low water 1725232 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.982848:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743840 low water 1725232 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.982863:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743840 low water 1725232 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.982874:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743840 low water 1725232 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.982880:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743840 low water 1725232 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.982918:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807756 low water 1693274 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.982922:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807756 low water 1693274 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.984212:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.984229:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.984239:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.984250:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.984255:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871672 low water 1661316 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.984285:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 999504 low water 1597400 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.984290:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 999504 low water 1597400 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.989252:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 999504 low water 1597400 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.989274:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 999504 low water 1597400 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.989286:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 999504 low water 1597400 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.989298:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 999504 low water 1597400 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.989310:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 999504 low water 1597400 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172519.996421:0:180303:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1894328 low water 1149988 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.137266:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.137284:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.137295:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.137305:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.137317:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.139556:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.139572:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.139584:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.139593:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.139603:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.141509:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.141524:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.141536:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.141548:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.141558:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.145669:0:180306:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.145685:0:180306:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.145694:0:180306:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -56296 low water 2125300 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.148271:0:180306:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -43484 low water 2118894 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.148278:0:180306:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -43484 low water 2118894 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.148378:0:180306:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.148389:0:180306:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.148398:0:180306:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.148408:0:180306:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.148417:0:180306:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.148421:0:180306:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.150475:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.150685:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.151224:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.151465:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.151687:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.151717:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.151733:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.151747:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.151760:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.151772:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 925988 low water 1634158 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172526.152605:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1892048 low water 1151128 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:3.1:1786172545.110532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180ba00. 00010000:00000010:3.1:1786172545.110552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98c00. 00010000:00000010:3.1:1786172545.110555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:3.1:1786172545.110562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459f00. 00010000:00000010:3.1:1786172545.110567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:3.1:1786172545.110571:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9400. 00010000:00000010:3.1:1786172545.110574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00000080:00000010:3.1:1786172545.129494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e02ed1da0. 00010000:00000010:0.1:1786172548.662497:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3500. 00010000:00000010:0.1:1786172548.662513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180b600. 00010000:00000010:0.1:1786172548.662518:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3000. 00010000:00000010:0.1:1786172548.662521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180b000. 00010000:00000010:0.1:1786172552.763550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180be00. 00010000:00000010:0.1:1786172552.785967:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180b800. 00000080:00000010:0.1:1786172555.188486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbfa40. 00000080:00000010:0.1:1786172555.188506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbfe60. 00010000:00000010:0.1:1786172555.188519:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3700. 00010000:00000010:0.1:1786172555.188527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180ba00. 00010000:00000010:0.1:1786172555.215540:0:2386:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3b00. 00010000:00000010:3.1:1786172555.273667:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9700. 00010000:00000010:3.1:1786172555.273687:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786172555.275518:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa500. 00010000:00000010:3.1:1786172555.275537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa200. 00010000:00000010:3.1:1786172555.275542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99800. 00000080:00000010:3.1:1786172555.355640:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca300. 00010000:00000010:3.1:1786172555.459489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:3.1:1786172555.487778:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401900. 00010000:00000010:0.1:1786172555.521004:0:180892:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e328bda00. 00010000:00000010:0.1:1786172565.468643:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9900. 00010000:00000010:3.1:1786172565.476618:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3000. 00010000:00000010:3.1:1786172565.476639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:0.1:1786172565.493642:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3c00. 00010000:00000010:0.1:1786172565.493658:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180b400. 00010000:00000010:3.1:1786172565.588830:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9800. 00010000:00000010:3.1:1786172565.588841:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00000080:00000010:0.1:1786172565.609434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776340. 00000800:00000200:3.1:1786172567.640722:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 167596 low water 2013354 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.640799:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 167596 low water 2013354 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.640815:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 167596 low water 2013354 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.640822:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 167596 low water 2013354 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.640872:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.646778:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.646963:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.647330:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.647532:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.651467:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295428 low water 1949438 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.651727:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295428 low water 1949438 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.651734:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295428 low water 1949438 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.651956:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.652260:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.652624:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.652811:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.653298:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.653580:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.653752:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.654666:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.654956:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.655352:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.655385:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.657784:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.657804:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.657818:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.657833:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.657845:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.657851:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.659594:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.659791:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.659820:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.659834:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.659844:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.659855:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.659865:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.659871:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.661260:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.661292:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.661306:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.661317:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.661328:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.661338:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.663689:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.663707:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.663720:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.663730:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.663740:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172567.670703:0:181472:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1445916 low water 1374194 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.991230:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.991247:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.991257:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.991266:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.991277:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.991710:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.991724:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.991734:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.991743:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.991752:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.992759:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.992778:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.992790:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.992801:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.992810:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.995788:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.995806:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.995816:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.995826:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.995840:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.995845:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.996737:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.996754:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.996764:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.996775:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.996786:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.996791:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.997275:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.997291:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.997302:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.997311:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.997321:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.998928:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.998950:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.998964:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.998973:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172570.998982:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.000494:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.000516:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.000528:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.000538:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.000550:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.001153:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.001169:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.001179:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.001189:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.001199:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.009442:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 2468572 low water 862866 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.967717:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 158296 low water 2018004 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.967739:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 158296 low water 2018004 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.967744:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 158296 low water 2018004 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.968124:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 158296 low water 2018004 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.968145:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 158296 low water 2018004 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.970454:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 232440 low water 1980932 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.970474:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 232440 low water 1980932 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.970479:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 232440 low water 1980932 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.970499:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 296356 low water 1948974 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.971121:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 296356 low water 1948974 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.971143:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 296356 low water 1948974 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.971511:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 296356 low water 1948974 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.971526:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 296356 low water 1948974 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.972118:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295076 low water 1949614 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.977941:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.977959:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.977970:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.977980:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.977990:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.978006:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.978792:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.978808:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.978819:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.978828:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.980833:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.980850:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.980862:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.980872:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 843476 low water 1675414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.983912:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382928 low water 1405688 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.983934:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382928 low water 1405688 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.983947:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382928 low water 1405688 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.983958:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382928 low water 1405688 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.983963:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382928 low water 1405688 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172571.987942:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1446844 low water 1373730 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.105992:0:181477:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.106027:0:181477:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.106040:0:181477:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.106050:0:181477:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.106060:0:181477:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.106070:0:181477:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.106075:0:181477:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.106854:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.106867:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.106876:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.106885:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.106894:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.109497:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.109510:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.109518:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.109528:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.109537:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.111793:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.111807:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.111817:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.111827:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.111836:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.111841:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.111869:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.114269:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.114638:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.114665:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.114676:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.114685:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.114696:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.114707:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615424 low water 1789440 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172574.116196:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1573748 low water 1310278 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:0.1:1786172574.366451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:3.1:1786172574.384400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:3.1:1786172581.765593:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172581.765602:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786172581.765607:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401200. 00010000:00000010:3.1:1786172581.765611:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e328bd000. 00010000:00000010:3.1:1786172581.765613:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9500. 00010000:00000010:3.1:1786172581.765616:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00000080:00000010:3.1:1786172581.811402:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776be0. 00010000:00000010:3.1:1786172586.198191:0:182009:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9600. 00010000:00000010:3.1:1786172586.198199:0:182009:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786172586.314960:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401700. 00010000:00000010:3.1:1786172586.321761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0933a200. 00000800:00000200:3.1:1786172586.803198:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -25004 low water 2109654 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.803403:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -25004 low water 2109654 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.804140:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -25004 low water 2109654 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.804411:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -25004 low water 2109654 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.804579:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -25004 low water 2109654 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.807307:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -25004 low water 2109654 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.807388:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -25004 low water 2109654 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.807404:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -25004 low water 2109654 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.807416:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -25004 low water 2109654 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.818479:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.818669:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.819080:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.819264:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.820320:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.820833:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.821823:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.822353:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.822837:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.823892:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.823940:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.823954:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.824827:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.824896:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 742840 low water 1725732 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.824944:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1062420 low water 1565942 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.824950:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1062420 low water 1565942 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.825528:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.828951:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.828969:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.828980:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.832265:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.832864:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.833081:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.833283:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.833309:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.833323:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.833334:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.833343:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382000 low water 1406152 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172586.833405:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1573748 low water 1310278 conn ffff9b9e03cc4000 blocked idle queued 00000080:00000010:0.1:1786172586.972425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296440. 00010000:00000010:0.1:1786172586.994241:0:181913:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9300. 00010000:00000010:0.1:1786172586.994265:0:181913:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:3.1:1786172587.001959:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9200. 00010000:00000010:3.1:1786172587.001967:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:3.1:1786172587.001970:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9d00. 00010000:00000010:3.1:1786172587.001974:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786172587.012990:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9b00. 00010000:00000010:3.1:1786172587.013006:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00000080:00000010:3.1:1786172587.158417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296020. 00000080:00000010:3.1:1786172587.158453:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32d1a940. 00010000:00000010:3.1:1786172587.197970:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aae00. 00010000:00000010:3.1:1786172587.198004:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99000. 00010000:00000010:3.1:1786172587.214695:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:3.1:1786172587.246683:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9500. 00010000:00000010:3.1:1786172587.246691:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786172588.198167:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9300. 00010000:00000010:3.1:1786172588.198185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786172588.205120:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa100. 00010000:00000010:3.1:1786172588.209434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:3.1:1786172590.624421:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9400. 00010000:00000010:3.1:1786172590.628410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00000800:00000200:3.1:1786172590.675711:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 230476 low water 1981914 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.675734:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 230476 low water 1981914 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.675753:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 230476 low water 1981914 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.675759:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 230476 low water 1981914 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.677406:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 230476 low water 1981914 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.677430:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 230476 low water 1981914 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.679765:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 230476 low water 1981914 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.679896:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 230476 low water 1981914 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.683954:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 230476 low water 1981914 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.693494:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679168 low water 1757568 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.693521:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679168 low water 1757568 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.693537:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679168 low water 1757568 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.693543:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679168 low water 1757568 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.693565:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743084 low water 1725610 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.694742:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743084 low water 1725610 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.694757:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743084 low water 1725610 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.694779:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743084 low water 1725610 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.697118:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 934832 low water 1629736 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.697141:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 934832 low water 1629736 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.697153:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 934832 low water 1629736 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.697158:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 934832 low water 1629736 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.697231:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998748 low water 1597778 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.697236:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998748 low water 1597778 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.701416:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998748 low water 1597778 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.701442:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998748 low water 1597778 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.701456:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998748 low water 1597778 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.701468:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998748 low water 1597778 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.702315:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126580 low water 1533862 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.702331:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126580 low water 1533862 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.702342:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126580 low water 1533862 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.702351:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126580 low water 1533862 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.702356:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126580 low water 1533862 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.702391:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.702396:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.704484:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.704504:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.704515:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.704523:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.704531:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.706643:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.706669:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.706681:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.706694:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.706707:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.708837:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.708862:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.708877:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1382244 low water 1406030 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.712377:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1637908 low water 1278198 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.747256:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39764 low water 2077270 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.747344:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295428 low water 1949438 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.747472:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.747490:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.747501:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.747511:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.747516:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.749462:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.749483:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.749499:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.749512:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.749525:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.749532:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.766148:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.766172:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.766187:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.766198:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.766207:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.766212:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.766240:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.767224:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.767239:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.767249:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.767258:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.767269:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.768618:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.768638:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.768650:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.768661:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.768672:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.769491:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.769506:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.769517:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.769528:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.769540:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.771541:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.771564:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.771576:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.771587:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.771597:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172590.781704:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1829412 low water 1182446 conn ffff9b9e03cc4000 blocked idle queued 00000080:00000010:3.1:1786172590.893394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca4c0. 00010000:00000010:3.1:1786172591.183960:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401300. 00010000:00000010:3.1:1786172591.183978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25397e00. 00010000:00000010:3.1:1786172591.183983:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa900. 00010000:00000010:3.1:1786172591.196141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98600. 00010000:00000010:3.1:1786172592.173589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9d00. 00010000:00000010:3.1:1786172592.173610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:3.1:1786172592.177655:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9000. 00010000:00000010:3.1:1786172592.177674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:3.1:1786172593.635344:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa900. 00010000:00000010:3.1:1786172593.635373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98600. 00010000:00000010:3.1:1786172595.841406:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa800. 00010000:00000010:3.1:1786172595.841419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98a00. 00000080:00000010:0.1:1786172618.872063:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca1a0. 00010000:00000010:0.1:1786172618.872080:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa000. 00010000:00000010:0.1:1786172618.872085:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00010000:00000010:3.1:1786172618.908497:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401100. 00010000:00000010:3.1:1786172618.908522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d827c00. 00010000:00000010:3.1:1786172618.925976:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa300. 00010000:00000010:3.1:1786172618.926001:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00000080:00000010:0.1:1786172618.984802:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf9e0. 00010000:00000010:3.1:1786172619.004504:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3f00. 00010000:00000010:3.1:1786172619.004523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d826e00. 00010000:00000010:3.1:1786172619.048037:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9000. 00010000:00000010:3.1:1786172619.048056:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98600. 00010000:00000010:3.1:1786172619.574458:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9300. 00010000:00000010:3.1:1786172619.574466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:0.1:1786172619.609407:0:2386:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3100. 00010000:00000010:0.1:1786172619.609416:0:2386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d826a00. 00010000:00000010:3.1:1786172621.899403:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9600. 00010000:00000010:3.1:1786172621.899481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:0.1:1786172621.930602:0:2394:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9700. 00010000:00000010:0.1:1786172621.930615:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00000080:00000010:3.1:1786172622.989671:0:182696:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296ac0. 00010000:00000010:3.1:1786172623.365712:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401500. 00010000:00000010:3.1:1786172623.365720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d800. 00000800:00000200:3.1:1786172625.107961:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.107984:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.107994:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.108004:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.108027:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.108032:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.117812:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.117832:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.117843:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.117866:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.122025:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.122052:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.122062:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.122071:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 549812 low water 1822246 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172625.122750:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1508552 low water 1342876 conn ffff9b9e03cc4000 blocked idle queued 00000080:00000010:0.1:1786172627.441692:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296460. 00010000:00000010:3.1:1786172627.579472:0:182700:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:3.1:1786172652.290830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00010000:00000010:3.1:1786172652.290843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:3.1:1786172652.290847:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa300. 00010000:00000010:3.1:1786172652.290851:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:3.1:1786172652.290853:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:3.1:1786172652.290855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99c00. 00000080:00000010:3.1:1786172652.340549:0:31:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296940. 00000080:00000010:3.1:1786172653.864894:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0221eb40. 00010000:00000010:0.1:1786172653.883425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:0.1:1786172653.883438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:0.1:1786172653.883444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9c00. 00010000:00000010:0.1:1786172653.883447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:0.1:1786172653.883451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:0.1:1786172653.883454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99c00. 00000080:00000010:0.1:1786172655.591010:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32d1f380. 00010000:00000010:3.1:1786172657.724632:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3800. 00010000:00000010:3.1:1786172657.724642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e250c0000. 00010000:00000010:3.1:1786172657.744932:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9900. 00010000:00000010:3.1:1786172657.744938:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786172657.811979:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9600. 00010000:00000010:3.1:1786172657.836593:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa800. 00010000:00000010:3.1:1786172657.836612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786172657.870991:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aab00. 00010000:00000010:3.1:1786172657.871011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99000. 00010000:00000010:3.1:1786172658.311401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401d00. 00010000:00000010:3.1:1786172658.311409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180b400. 00000080:00000010:3.1:1786172662.181503:0:183137:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296b80. 00010000:00000010:0.1:1786172662.274413:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3e00. 00010000:00000010:0.1:1786172662.274431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da0ea00. 00010000:00000010:3.1:1786172662.288074:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3c00. 00010000:00000010:3.1:1786172662.294512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da0e000. 00000080:00000010:3.1:1786172662.410826:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbfe60. 00010000:00000010:3.1:1786172663.877679:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9d00. 00010000:00000010:3.1:1786172663.877691:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172663.877695:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9500. 00010000:00000010:3.1:1786172663.877697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00000080:00000010:3.1:1786172667.752633:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf040. 00000080:00000010:3.1:1786172667.765495:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0221e300. 00010000:00000010:3.1:1786172667.800512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:3.1:1786172667.800528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99400. 00010000:00000010:3.1:1786172667.800531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:3.1:1786172667.800536:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aab00. 00010000:00000010:3.1:1786172667.800539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99000. 00010000:00000010:3.1:1786172667.800541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:3.1:1786172667.800544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00000080:00000010:3.1:1786172667.825769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776ca0. 00000080:00000010:3.1:1786172667.830206:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02175280. 00010000:00000010:3.1:1786172668.989404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3e00. 00010000:00000010:3.1:1786172668.989430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da0ea00. 00010000:00000010:3.1:1786172668.989437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9400. 00010000:00000010:3.1:1786172668.989441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:3.1:1786172670.074392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401d00. 00010000:00000010:3.1:1786172670.077486:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:3.1:1786172670.077498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26069800. 00000080:00000010:3.1:1786172670.149498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296f00. 00010000:00000010:3.1:1786172670.157562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:3.1:1786172670.157572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00010000:00000010:3.1:1786172670.163011:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaf00. 00010000:00000010:3.1:1786172670.180453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:3.1:1786172670.180469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98a00. 00010000:00000010:3.1:1786172670.206402:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:3.1:1786172670.206418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00010000:00000010:3.1:1786172670.349396:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401800. 00010000:00000010:3.1:1786172670.349412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26069600. 00010000:00000010:3.1:1786172670.362394:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:3.1:1786172670.388349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00010000:00000010:3.1:1786172670.801408:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9a00. 00010000:00000010:3.1:1786172670.801419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786172670.877482:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9300. 00010000:00000010:3.1:1786172670.877500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00000800:00000200:3.1:1786172670.901949:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.901965:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.901976:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.901987:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.901999:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.904279:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.904300:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.904312:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.904326:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.904336:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.904342:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.904382:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.904404:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551508 low water 1821398 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.907187:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.907211:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.907222:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.907232:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.907243:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.907250:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.908649:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.908670:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.908680:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.908691:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.908702:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.910588:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.910612:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.910622:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.910632:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.910640:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.913009:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.913032:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.913040:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.913063:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.913073:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.915131:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.915148:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.915160:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.915172:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.915183:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.922317:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.922334:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.922345:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.922355:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.922384:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.922390:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172670.922476:0:2396:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 2468572 low water 862866 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172671.523563:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 133288 low water 2030508 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172671.523578:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 133288 low water 2030508 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172671.523588:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 133288 low water 2030508 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172671.523592:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 133288 low water 2030508 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172671.534077:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 997224 low water 1598540 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172671.534089:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 997224 low water 1598540 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172671.534209:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 3298200 low water 448052 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.215206:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -48764 low water 2121534 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.215221:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -48764 low water 2121534 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.215232:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -48764 low water 2121534 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.215242:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -48764 low water 2121534 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.217015:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.217049:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.217062:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.217073:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.217079:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.218292:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.218309:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.218319:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.218330:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.219433:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.219447:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.219458:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.219468:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.220499:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.220513:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.220522:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -11692 low water 2102998 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.231063:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615976 low water 1789164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.231097:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615976 low water 1789164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.231110:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615976 low water 1789164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.231117:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615976 low water 1789164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.232429:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615976 low water 1789164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.232443:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615976 low water 1789164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.232453:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615976 low water 1789164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.234457:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615976 low water 1789164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.234474:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615976 low water 1789164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.242477:0:31:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871640 low water 1661332 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.242499:0:31:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871640 low water 1661332 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.242513:0:31:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871640 low water 1661332 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.242519:0:31:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871640 low water 1661332 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172672.242682:0:31:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 3682976 low water 255664 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.758511:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.758536:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.758550:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.758562:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.758572:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.758578:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.761327:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.761345:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.761356:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.761366:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.761384:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.764374:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.764392:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.764405:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.764418:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.764430:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.765079:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.765098:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.765109:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.765121:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.765134:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.765835:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.765850:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.765859:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.765869:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.765878:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.767762:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.767781:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.767794:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.767808:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.767824:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.770664:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.770682:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.770694:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.770704:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.770714:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172673.771478:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 2085076 low water 1054614 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:0.1:1786172674.218439:0:2386:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:0.1:1786172674.218455:0:2386:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:0.1:1786172674.218467:0:2386:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.223240:0:183870:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 167596 low water 2013354 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.223263:0:183870:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 167596 low water 2013354 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.223275:0:183870:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 167596 low water 2013354 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.223280:0:183870:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 167596 low water 2013354 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.223345:0:183870:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.223395:0:183870:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.223400:0:183870:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.224504:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.225892:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.225929:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.225946:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.225958:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.225969:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.229194:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.229221:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.229234:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.229246:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.232361:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.232387:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.232401:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.232414:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.232420:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.232991:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.233006:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.233028:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.233038:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.234353:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.234375:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.234388:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.234398:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.238279:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.238300:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.238312:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.238323:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.238713:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.238726:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.238738:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.238748:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.239698:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1509832 low water 1342236 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172674.258386:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1956960 low water 1118672 conn ffff9b9e03cc4000 blocked idle queued 00000080:00000010:0.1:1786172674.902221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca660. 00010000:00000010:3.1:1786172675.510076:0:183874:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9400. 00010000:00000010:3.1:1786172675.510103:0:183874:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00000080:00000010:0.1:1786172679.958417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca320. 00010000:00000010:0.1:1786172703.546660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99000. 00010000:00000010:0.1:1786172703.546685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98a00. 00010000:00000010:0.1:1786172703.546689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99400. 00010000:00000010:0.1:1786172703.546695:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401000. 00010000:00000010:0.1:1786172703.546701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26068e00. 00010000:00000010:0.1:1786172703.546708:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3000. 00010000:00000010:0.1:1786172703.546713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26068200. 00010000:00000010:3.1:1786172705.463254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786172705.463279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172705.463284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98c00. 00010000:00000010:3.1:1786172705.463290:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9700. 00010000:00000010:3.1:1786172705.463295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786172705.463299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401800. 00010000:00000010:3.1:1786172705.463302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1da0fa00. 00000080:00000010:3.1:1786172705.531505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776700. 00010000:00000010:3.1:1786172709.950386:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3f00. 00010000:00000010:3.1:1786172709.950400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e317cc000. 00010000:00000010:3.1:1786172710.002033:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3300. 00010000:00000010:3.1:1786172710.002054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e317cd000. 00010000:00000010:3.1:1786172712.722581:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9900. 00010000:00000010:3.1:1786172712.722590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:3.1:1786172729.626424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa500. 00010000:00000010:3.1:1786172729.626447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00010000:00000010:3.1:1786172729.649466:0:31:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaa00. 00010000:00000010:3.1:1786172729.672412:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa600. 00010000:00000010:3.1:1786172729.672430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00010000:00000010:3.1:1786172743.524845:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9800. 00010000:00000010:3.1:1786172743.524858:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:0.1:1786172743.602865:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9300. 00010000:00000010:0.1:1786172743.602872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172743.630681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26069c00. 00010000:00000010:3.1:1786172744.058870:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3300. 00010000:00000010:3.1:1786172744.058893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26068400. 00010000:00000010:3.1:1786172744.066788:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3f00. 00010000:00000010:3.1:1786172744.066803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26069800. 00010000:00000010:3.1:1786172744.125387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3900. 00010000:00000010:3.1:1786172744.125403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e26069400. 00000080:00000010:3.1:1786172744.994431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf860. 00010000:00000010:3.1:1786172749.888597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401100. 00010000:00000010:3.1:1786172749.888618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd800. 00010000:00000010:3.1:1786172754.709615:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401000. 00010000:00000010:3.1:1786172754.709633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fcc00. 00010000:00000010:3.1:1786172754.713852:0:2395:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401c00. 00010000:00000010:3.1:1786172754.713871:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fce00. 00010000:00000010:3.1:1786172761.923412:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa100. 00010000:00000010:3.1:1786172761.923427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00010000:00000010:3.1:1786172761.967101:0:1008:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa700. 00010000:00000010:3.1:1786172761.968570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa100. 00010000:00000010:3.1:1786172761.968577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98600. 00010000:00000010:3.1:1786172761.998397:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa300. 00010000:00000010:3.1:1786172761.998407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00010000:00000010:3.1:1786172762.963419:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa300. 00010000:00000010:3.1:1786172762.963442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00010000:00000010:3.1:1786172762.984560:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa100. 00010000:00000010:0.1:1786172763.559240:0:185172:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401500. 00010000:00000010:0.1:1786172763.559252:0:185172:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fc200. 00000080:00000010:3.1:1786172820.247722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca960. 00010000:00000010:3.1:1786172822.595407:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401700. 00010000:00000010:3.1:1786172822.595422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd400. 00010000:00000010:3.1:1786172822.609700:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9900. 00010000:00000010:3.1:1786172822.609710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00000080:00000010:3.1:1786172826.470403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09296920. 00010000:00000010:0.1:1786172883.762418:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9a00. 00010000:00000010:0.1:1786172883.762435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:0.1:1786172883.762440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3100. 00010000:00000010:0.1:1786172883.762444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3178b600. 00010000:00000010:3.1:1786172885.233432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:3.1:1786172885.255416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:3.1:1786172885.295599:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00000080:00000010:0.1:1786172885.537537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf860. 00010000:00000010:3.1:1786172887.693816:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9b00. 00010000:00000010:3.1:1786172887.693830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3178b800. 00010000:00000010:3.1:1786172889.135761:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401900. 00010000:00000010:3.1:1786172889.135778:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31a05e00. 00000080:00000010:0.1:1786172943.127020:0:185826:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776340. 00010000:00000010:0.1:1786172945.129435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3500. 00010000:00000010:0.1:1786172945.129452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942800. 00010000:00000010:0.1:1786172945.141627:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3c00. 00010000:00000010:0.1:1786172945.155346:0:186059:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943800. 00010000:00000010:3.1:1786172945.487616:0:186059:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3c00. 00010000:00000010:3.1:1786172945.487628:0:186059:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943800. 00010000:00000010:3.1:1786172945.533948:0:2396:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9900. 00010000:00000010:3.1:1786172945.533968:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786172945.553823:0:186059:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401a00. 00010000:00000010:3.1:1786172945.553838:0:186059:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e013fd600. 00000800:00000200:3.1:1786172945.597884:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.597904:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.597918:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.597931:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.597944:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.597950:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.600479:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.600498:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.600510:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.600523:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.600534:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.601234:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.601254:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.601268:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.601281:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.601292:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.601986:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.602004:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.602016:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.602027:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 91308 low water 2051498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786172945.616290:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 2212908 low water 990698 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:3.1:1786172951.101668:0:2396:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39401700. 00010000:00000010:3.1:1786173014.847403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:3.1:1786173014.847420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f40a00. 00010000:00000010:3.1:1786173014.847423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:3.1:1786173014.847427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3f00. 00010000:00000010:3.1:1786173014.847433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180bc00. 00010000:00000010:3.1:1786173014.847437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:3.1:1786173014.847440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00000080:00000010:3.1:1786173014.868410:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776ec0. 00000080:00000010:3.1:1786173014.874468:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02177380. 00010000:00000010:3.1:1786173015.674401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98a00. 00000080:00000010:0.1:1786173019.592450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776d60. 00010000:00000010:0.1:1786173019.592469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9100. 00010000:00000010:0.1:1786173019.592475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:0.1:1786173019.617557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3000. 00010000:00000010:3.1:1786173019.644586:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3600. 00010000:00000010:3.1:1786173019.644610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f40400. 00010000:00000010:0.1:1786173019.665399:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3900. 00010000:00000010:0.1:1786173019.665417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f40000. 00000080:00000010:0.1:1786173019.719530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbfea0. 00010000:00000010:3.1:1786173019.778819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:3.1:1786173020.158945:0:186684:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:3.1:1786173020.158953:0:186684:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98a00. 00000800:00000200:3.1:1786173020.246251:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.246274:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.246284:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.246295:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.246301:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.247620:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.247636:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.247647:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.247660:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.247665:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.248617:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.248633:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.248644:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.248655:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.251025:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.251050:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.251059:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.251068:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487420 low water 1853442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.271949:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1701824 low water 1246240 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.274936:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23908 low water 2109106 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.275334:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23908 low water 2109106 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.276690:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23908 low water 2109106 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.277115:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23908 low water 2109106 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.277486:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23908 low water 2109106 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.277519:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23908 low water 2109106 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.277532:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23908 low water 2109106 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.277542:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23908 low water 2109106 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.277554:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23908 low water 2109106 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.280177:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551336 low water 1821484 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.280201:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551336 low water 1821484 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.280212:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551336 low water 1821484 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.280223:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551336 low water 1821484 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.280229:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551336 low water 1821484 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.285511:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551336 low water 1821484 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.285533:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551336 low water 1821484 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.285544:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551336 low water 1821484 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.285554:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551336 low water 1821484 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.288893:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1062420 low water 1565942 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.288917:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1062420 low water 1565942 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.288929:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1062420 low water 1565942 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.288940:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1062420 low water 1565942 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.288946:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1062420 low water 1565942 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.293854:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.293883:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173020.293950:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1573748 low water 1310278 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:3.1:1786173020.779986:0:2395:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaf00. 00010000:00000010:3.1:1786173020.779993:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:3.1:1786173020.813701:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aae00. 00010000:00000010:3.1:1786173020.813708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00000080:00000010:0.1:1786173023.312425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117767a0. 00010000:00000010:0.1:1786173023.368417:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3000. 00010000:00000010:0.1:1786173023.368434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024ee00. 00000800:00000200:3.1:1786173023.784176:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23736 low water 2109020 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.784194:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23736 low water 2109020 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.784204:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23736 low water 2109020 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.784214:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23736 low water 2109020 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.784225:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23736 low water 2109020 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.785768:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23736 low water 2109020 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.785796:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23736 low water 2109020 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.785808:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23736 low water 2109020 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.785818:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23736 low water 2109020 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.793462:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 40180 low water 2077062 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.793487:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 40180 low water 2077062 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.793492:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 40180 low water 2077062 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.795300:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.795323:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.795335:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.795346:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.795357:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.795363:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.795387:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.796738:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.796763:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.796777:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.796788:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.796806:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.797813:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.797830:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.797843:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.797854:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.797865:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.797871:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.797895:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.800422:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1637664 low water 1278320 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.806045:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.806101:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.806848:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.806881:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.806894:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.806906:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.806917:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.806926:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.807897:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.807915:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.807926:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.807937:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.807948:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.808306:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.808316:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.808322:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.808328:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.808334:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.809083:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.809439:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.809700:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.809901:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.809927:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.809940:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.809953:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24152 low water 2109228 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.810770:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.810792:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.810804:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.810814:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.810825:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.810830:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 615008 low water 1789648 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.810863:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.812399:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.812416:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.812428:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.812439:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.812451:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:0.1:1786173023.813784:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:0.1:1786173023.813812:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:0.1:1786173023.813824:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:0.1:1786173023.813834:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:0.1:1786173023.813845:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.816444:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.816472:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.816485:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.816497:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.816508:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.817137:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.817154:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.817166:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173023.818016:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1637664 low water 1278320 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.448802:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 139472 low water 2027416 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.448825:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 139472 low water 2027416 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.448837:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 139472 low water 2027416 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.448847:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 139472 low water 2027416 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.448853:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 139472 low water 2027416 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.451255:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.451286:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.451305:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.451314:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 359344 low water 1917480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.452896:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.452914:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.452926:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.452937:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.452942:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.454902:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.454921:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.454931:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.454942:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.454948:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.456875:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.456895:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.456906:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.456915:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.457790:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.457806:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.457817:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.457827:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.458908:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.458928:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.458938:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.458949:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.461552:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.461569:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.461580:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.461591:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.461601:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.461607:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.470815:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1062420 low water 1565942 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173024.470889:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1829412 low water 1182446 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.128673:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487328 low water 1853488 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.128697:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487328 low water 1853488 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.128711:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487328 low water 1853488 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.128725:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487328 low water 1853488 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.128739:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487328 low water 1853488 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.128746:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487328 low water 1853488 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.128798:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.129498:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.129516:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.129530:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.129542:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.129556:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.130416:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.130431:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.130443:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.130456:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.130468:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.130719:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.130738:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 934588 low water 1629858 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.130795:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.137609:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 2276824 low water 958740 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.149452:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23424 low water 2108864 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.149473:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23424 low water 2108864 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.149485:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23424 low water 2108864 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.149497:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23424 low water 2108864 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.149508:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -23424 low water 2108864 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.150742:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.150762:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.150773:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.150790:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.150801:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.150807:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.151465:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.151485:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.151496:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.151507:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.151518:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.152007:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.152024:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.152036:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.152047:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.152058:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 679652 low water 1757326 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.154439:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743568 low water 1725368 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.154464:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743568 low water 1725368 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.154476:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743568 low water 1725368 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.154488:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743568 low water 1725368 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.154499:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743568 low water 1725368 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.154507:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743568 low water 1725368 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.156444:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 743568 low water 1725368 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.163008:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 2724236 low water 735034 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.696870:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231512 low water 1981396 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.696987:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487176 low water 1853564 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.696993:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 487176 low water 1853564 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699109:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699129:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699140:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699151:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699164:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699169:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 551092 low water 1821606 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699197:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699906:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699922:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699931:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699941:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.699951:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.701051:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.701069:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.701079:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.701088:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.701096:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.701665:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.701911:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.701927:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.701938:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.701948:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.701958:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.704813:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.704834:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.704845:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.704856:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.704866:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.704872:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.706642:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1829412 low water 1182446 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.709134:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24024 low water 2109164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.709150:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24024 low water 2109164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.709163:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24024 low water 2109164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.709173:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24024 low water 2109164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.709181:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24024 low water 2109164 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.712139:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39892 low water 2077206 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.712163:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39892 low water 2077206 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.712174:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39892 low water 2077206 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.712184:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39892 low water 2077206 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.712193:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39892 low water 2077206 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.712199:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39892 low water 2077206 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.712241:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295556 low water 1949374 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.712282:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.716081:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.716104:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.716115:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.716125:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.716136:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.716612:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.716637:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.716650:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.716661:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.716670:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.717513:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.717530:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.717539:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.717548:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.717557:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.718028:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.718043:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.718055:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.718064:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.718074:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.719678:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.719699:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.719711:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.719720:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.719730:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.720454:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.720469:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.720479:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.720488:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.720497:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.722914:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.722934:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.722944:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.722953:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998632 low water 1597836 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173025.728830:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 3235692 low water 479306 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.280111:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39764 low water 2077270 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.280122:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39764 low water 2077270 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.280128:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39764 low water 2077270 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.280134:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39764 low water 2077270 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.280140:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39764 low water 2077270 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.280143:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 39764 low water 2077270 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.281860:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 678924 low water 1757690 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.281882:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 678924 low water 1757690 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.281893:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 678924 low water 1757690 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.281904:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 678924 low water 1757690 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.281915:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 678924 low water 1757690 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.281921:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 678924 low water 1757690 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.283717:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.283739:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.283751:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.283763:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.283774:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.283780:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.283803:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.283949:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1190252 low water 1502026 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.283972:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.284345:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.284358:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.284367:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.284375:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.284385:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.286077:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.286097:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.286108:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.286117:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.286125:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.289807:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 2212908 low water 990698 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.752894:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.752913:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.752926:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.752936:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.752947:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.753840:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.753872:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.753887:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.753898:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.753909:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.753920:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173026.756644:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 2468572 low water 862866 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.961314:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -17144 low water 2105724 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.961324:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -17144 low water 2105724 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.961418:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 794584 low water 1699860 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.961432:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 794584 low water 1699860 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.961445:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 794584 low water 1699860 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.961457:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 794584 low water 1699860 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.961468:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 794584 low water 1699860 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.961473:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 794584 low water 1699860 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.961500:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 932812 low water 1630746 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.961520:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.962725:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.962815:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.962824:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.962832:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.962841:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.964648:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.964663:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.964674:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.964685:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.964695:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.965245:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.965259:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.965268:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.965276:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.965284:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.966108:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.966124:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.966133:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.966142:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.966150:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.967359:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.967378:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.967389:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.967398:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.967406:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1015120 low water 1589592 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.974422:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1052192 low water 1571056 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.974444:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1052192 low water 1571056 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.974453:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1052192 low water 1571056 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.974462:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1052192 low water 1571056 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.974471:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1052192 low water 1571056 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.974477:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1052192 low water 1571056 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.976892:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1765496 low water 1214404 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.982592:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 997952 low water 1598176 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.982618:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 997952 low water 1598176 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.982630:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 997952 low water 1598176 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.982651:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 997952 low water 1598176 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.982664:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 997952 low water 1598176 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.982670:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 997952 low water 1598176 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.984536:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1061868 low water 1566218 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.984568:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1061868 low water 1566218 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.984582:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1061868 low water 1566218 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.984594:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1061868 low water 1566218 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.984617:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1061868 low water 1566218 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.984624:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1061868 low water 1566218 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.988098:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1125784 low water 1534260 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.988119:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1125784 low water 1534260 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.988132:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1125784 low water 1534260 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.988142:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1125784 low water 1534260 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.988153:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1125784 low water 1534260 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.988159:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1125784 low water 1534260 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173027.989515:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 2980628 low water 606838 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:3.1:1786173028.247412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786173028.284093:0:186691:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:3.1:1786173052.317391:0:187115:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024f400. 00010000:00000010:3.1:1786173052.317396:0:187115:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:3.1:1786173052.317401:0:187115:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3900. 00010000:00000010:3.1:1786173052.317406:0:187115:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02f40000. 00010000:00000010:3.1:1786173052.317416:0:187115:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9a00. 00010000:00000010:3.1:1786173052.317419:0:187115:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00000080:00000010:3.1:1786173052.376403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf500. 00000080:00000010:3.1:1786173053.655923:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a28f380. 00000080:00000010:0.1:1786173055.020396:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0221b180. 00010000:00000010:3.1:1786173055.058537:0:8227:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98800. 00010000:00000010:0.1:1786173057.627731:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9600. 00010000:00000010:0.1:1786173057.627738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786173067.082413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:3.1:1786173067.849564:0:2396:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa300. 00010000:00000010:3.1:1786173067.849578:0:2396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98600. 00010000:00000010:3.1:1786173067.857826:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:3.1:1786173067.857840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98000. 00010000:00000010:0.1:1786173071.973549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaf00. 00010000:00000010:0.1:1786173071.973557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:0.1:1786173073.138951:0:187858:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786173075.363869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e252a8000. 00010000:00000010:3.1:1786173079.001448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00010000:00000010:0.1:1786173088.182635:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00000800:00000200:3.1:1786173090.144323:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 621284 low water 1786510 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.145253:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 621284 low water 1786510 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.146054:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 621284 low water 1786510 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.147027:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 621284 low water 1786510 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.147649:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 621284 low water 1786510 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.149060:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 621284 low water 1786510 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.149112:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 621284 low water 1786510 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.149128:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 621284 low water 1786510 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.149138:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 621284 low water 1786510 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.149157:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 621284 low water 1786510 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.149168:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 621284 low water 1786510 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.149260:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 685200 low water 1754552 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.149280:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 813032 low water 1690636 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173090.149333:0:1028:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1572468 low water 1310918 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:0.1:1786173097.905503:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9300. 00010000:00000010:0.1:1786173097.905516:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99600. 00010000:00000010:3.1:1786173114.009748:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3e00. 00010000:00000010:3.1:1786173114.009757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e06154400. 00010000:00000010:3.1:1786173118.666435:0:187869:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98600. 00010000:00000010:3.1:1786173118.706398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99400. 00010000:00000010:3.1:1786173118.706405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98e00. 00010000:00000010:3.1:1786173118.706408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99000. 00010000:00000010:3.1:1786173118.706411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:3.1:1786173118.706413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99c00. 00010000:00000010:3.1:1786173118.706415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99600. 00010000:00000010:3.1:1786173123.032403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:3.1:1786173123.032425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b7fc00. 00010000:00000010:3.1:1786173123.032428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b7f400. 00010000:00000010:3.1:1786173123.032432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9c00. 00010000:00000010:3.1:1786173123.032435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:3.1:1786173123.032438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230100. 00010000:00000010:3.1:1786173123.032441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00000080:00000010:3.1:1786173123.056411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09323920. 00010000:00000010:3.1:1786173123.109468:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230a00. 00010000:00000010:3.1:1786173123.109477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b7fe00. 00010000:00000010:0.1:1786173123.122759:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230900. 00010000:00000010:0.1:1786173123.122768:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b7f800. 00010000:00000010:3.1:1786173123.342515:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99600. 00010000:00000010:0.1:1786173123.876271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318eb000. 00010000:00000010:0.1:1786173123.888421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318ea000. 00010000:00000010:3.1:1786173124.321474:0:187876:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa000. 00010000:00000010:3.1:1786173124.321481:0:187876:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98400. 00000080:00000010:3.1:1786173124.331973:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32d1c200. 00010000:00000010:0.1:1786173126.509515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00000800:00000200:3.1:1786173128.006876:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.006893:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.006904:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.006915:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.006927:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.007752:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.007766:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.007775:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.007784:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.007803:0:2395:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.011257:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.011275:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.011287:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.011300:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.011311:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.012213:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.012226:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.012237:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.012247:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.012257:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.013224:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.013243:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.013255:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.013265:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.013273:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.014212:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.014229:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.014240:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.014251:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.014263:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -51128 low water 2122716 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.016021:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.016264:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.016457:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.016484:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.016496:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.016507:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.016516:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.017445:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.017457:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.017466:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.017475:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.017484:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.017492:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.017608:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.018677:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.018691:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.018700:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.018709:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.018719:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.019745:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.019761:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.019772:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.019781:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.019790:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.021207:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.021224:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.021236:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.021246:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.021256:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.024315:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 934588 low water 1629858 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.024334:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 934588 low water 1629858 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.024347:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 934588 low water 1629858 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.024352:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 934588 low water 1629858 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.024453:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 933308 low water 1630498 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.025990:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 2595124 low water 799590 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.350010:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -38220 low water 2116262 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.351335:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.351717:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.351914:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.352109:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.352290:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.352629:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.352840:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.353024:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.353053:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.353067:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.353077:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.353088:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.353098:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.353768:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.353788:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.353799:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.353808:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.354689:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.354706:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.354717:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.354729:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 741560 low water 1726372 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.359786:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 805476 low water 1694414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.359811:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 805476 low water 1694414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.359823:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 805476 low water 1694414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.359835:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 805476 low water 1694414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.359842:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 805476 low water 1694414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173128.359949:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1764216 low water 1215044 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:0.1:1786173130.240432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318ebc00. 00000800:00000200:3.1:1786173131.027175:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 742840 low water 1725732 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.027195:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 742840 low water 1725732 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.027210:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 742840 low water 1725732 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.027226:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 742840 low water 1725732 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.027237:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 742840 low water 1725732 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.027326:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 806756 low water 1693774 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.027386:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 870672 low water 1661816 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.027416:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.029335:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.029375:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.029394:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.029416:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.031416:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.031436:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.031449:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.031459:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.034028:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.034053:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.034063:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.034073:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 998504 low water 1597900 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.035443:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.035461:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.035476:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.035493:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.035501:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.035557:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.035565:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173131.035594:0:187875:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1509832 low water 1342236 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:3.1:1786173136.957955:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3100. 00010000:00000010:3.1:1786173136.957964:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:0.1:1786173142.243408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99e00. 00010000:00000010:0.1:1786173142.243423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e318ebe00. 00010000:00000010:0.1:1786173142.243427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:0.1:1786173142.243431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024f000. 00010000:00000010:0.1:1786173142.243435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9700. 00010000:00000010:0.1:1786173142.243439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:0.1:1786173142.243442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230f00. 00010000:00000010:0.1:1786173142.243445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b7e400. 00000080:00000010:0.1:1786173142.297514:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e117764a0. 00000080:00000010:3.1:1786173142.933629:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0221b9c0. 00010000:00000010:3.1:1786173142.999597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9800. 00010000:00000010:3.1:1786173142.999611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786173143.006411:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaa00. 00010000:00000010:3.1:1786173143.046956:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aab00. 00010000:00000010:3.1:1786173143.046980:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343c00. 00010000:00000010:3.1:1786173143.171469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342600. 00010000:00000010:0.1:1786173143.665478:0:187885:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3000. 00010000:00000010:0.1:1786173143.665507:0:187885:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024f200. 00010000:00000010:0.1:1786173143.686093:0:2394:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3b00. 00010000:00000010:0.1:1786173143.686108:0:2394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024ea00. 00000800:00000200:3.1:1786173144.520775:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 7644 low water 2093330 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.520797:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 7644 low water 2093330 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.520809:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 7644 low water 2093330 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.520818:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 7644 low water 2093330 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.520828:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 7644 low water 2093330 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.520833:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 7644 low water 2093330 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.520895:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 760736 low water 1716784 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.522375:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.522393:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.522403:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.522415:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.522424:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.522430:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1126336 low water 1533984 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173144.522497:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1445916 low water 1374194 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:0.1:1786173145.828731:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3b00. 00010000:00000010:0.1:1786173145.828739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e02e36c00. 00010000:00000010:3.1:1786173146.085413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342600. 00010000:00000010:3.1:1786173146.085423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342200. 00010000:00000010:3.1:1786173146.085426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342c00. 00010000:00000010:3.1:1786173146.085431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343400. 00010000:00000010:3.1:1786173146.085433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343600. 00010000:00000010:3.1:1786173146.085436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342e00. 00010000:00000010:3.1:1786173146.085439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343800. 00010000:00000010:3.1:1786173146.085442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342400. 00010000:00000010:3.1:1786173149.721416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:3.1:1786173149.721427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024ee00. 00010000:00000010:3.1:1786173149.721444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024ec00. 00010000:00000010:3.1:1786173149.721450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aab00. 00010000:00000010:3.1:1786173149.721455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343c00. 00010000:00000010:3.1:1786173149.721457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaa00. 00010000:00000010:3.1:1786173149.721461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00000080:00000010:3.1:1786173149.820456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbfbe0. 00010000:00000010:3.1:1786173150.251212:0:187895:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa800. 00010000:00000010:3.1:1786173150.251219:0:187895:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343a00. 00010000:00000010:3.1:1786173150.255454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9200. 00010000:00000010:3.1:1786173150.297707:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa800. 00010000:00000010:3.1:1786173150.297719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343a00. 00010000:00000010:3.1:1786173150.511855:0:187897:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342400. 00010000:00000010:0.1:1786173150.619848:0:187897:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aab00. 00010000:00000010:0.1:1786173150.619861:0:187897:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343c00. 00010000:00000010:3.1:1786173152.819433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9600. 00010000:00000010:3.1:1786173152.819452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786173153.044417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786173153.044432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:3.1:1786173153.044435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:3.1:1786173153.044438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00010000:00000010:3.1:1786173153.044440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786173153.044444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974000. 00010000:00000010:3.1:1786173153.044448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:3.1:1786173156.827733:0:187907:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31e8b000. 00010000:00000010:3.1:1786173156.827741:0:187907:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9e00. 00010000:00000010:3.1:1786173156.827746:0:187907:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00000080:00000010:3.1:1786173156.900474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09323fe0. 00000080:00000010:3.1:1786173156.900486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf420. 00010000:00000010:3.1:1786173156.900492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:3.1:1786173156.900496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31e8a400. 00010000:00000010:3.1:1786173156.900499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:3.1:1786173156.900502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:3.1:1786173156.900505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024e600. 00010000:00000010:3.1:1786173156.900507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786173156.900510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024fa00. 00010000:00000010:3.1:1786173156.900512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:3.1:1786173156.900517:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9000. 00010000:00000010:3.1:1786173156.900520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1024f600. 00010000:00000010:3.1:1786173156.900524:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa900. 00010000:00000010:3.1:1786173156.900527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99200. 00010000:00000010:3.1:1786173156.900530:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9400. 00010000:00000010:3.1:1786173156.900532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:3.1:1786173156.900535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99800. 00010000:00000010:3.1:1786173156.900538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa200. 00010000:00000010:3.1:1786173156.900541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b99a00. 00010000:00000010:3.1:1786173156.900544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3700. 00010000:00000010:3.1:1786173156.900548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3180ae00. 00000080:00000010:3.1:1786173156.908443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11776260. 00010000:00000010:3.1:1786173162.543482:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3000. 00010000:00000010:3.1:1786173162.543497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975400. 00010000:00000010:3.1:1786173162.557156:0:1030:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3600. 00010000:00000010:3.1:1786173162.557174:0:1030:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:3.1:1786173163.877402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98200. 00010000:00000010:3.1:1786173163.877421:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230d00. 00010000:00000010:3.1:1786173163.877425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e32b7e200. 00010000:00000010:3.1:1786173163.877556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230f00. 00010000:00000010:3.1:1786173163.877562:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f970200. 00000080:00000010:3.1:1786173163.877566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcac60. 00010000:00000010:3.1:1786173166.826140:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230100. 00010000:00000010:3.1:1786173166.826157:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f970800. 00010000:00000010:3.1:1786173166.836608:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaf00. 00010000:00000010:3.1:1786173166.974591:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaf00. 00010000:00000010:3.1:1786173166.974608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343a00. 00010000:00000010:3.1:1786173167.012692:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa600. 00010000:00000010:3.1:1786173167.034717:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaf00. 00010000:00000010:3.1:1786173167.034734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343200. 00010000:00000010:0.1:1786173167.155415:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaf00. 00010000:00000010:0.1:1786173167.155470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343200. 00010000:00000010:0.1:1786173167.182486:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3400. 00010000:00000010:0.1:1786173168.037873:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa700. 00010000:00000010:0.1:1786173168.037884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342800. 00010000:00000010:3.1:1786173169.001433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aae00. 00010000:00000010:3.1:1786173169.001441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98800. 00010000:00000010:3.1:1786173169.001444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:3.1:1786173169.001447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e25b98c00. 00000080:00000010:0.1:1786173169.076102:0:1348:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbf680. 00010000:00000010:0.1:1786173169.528207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230e00. 00010000:00000010:0.1:1786173169.528214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943400. 00010000:00000010:3.1:1786173169.686952:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230400. 00010000:00000010:3.1:1786173169.686962:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943e00. 00010000:00000010:3.1:1786173171.058693:0:2395:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9a00. 00010000:00000010:3.1:1786173171.058710:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786173172.117558:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3400. 00010000:00000010:0.1:1786173172.117567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00000800:00000200:3.1:1786173175.974648:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -47860 low water 2121082 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.974674:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -49140 low water 2121722 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.974688:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -50420 low water 2122362 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.975170:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 27392 low water 2083456 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.975198:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 26112 low water 2084096 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.975210:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 24832 low water 2084736 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.975215:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 24832 low water 2084736 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.975578:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 24832 low water 2084736 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.975596:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 23552 low water 2085376 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.975608:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 22272 low water 2086016 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.976153:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 60624 low water 2066840 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.976171:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 59344 low water 2067480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.976192:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 59344 low water 2067480 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.976200:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 58064 low water 2068120 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.976220:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 86188 low water 2054058 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.976552:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 86188 low water 2054058 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.976566:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 86188 low water 2054058 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.976582:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 84908 low water 2054698 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.977213:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 84908 low water 2054698 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.977233:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 83628 low water 2055338 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.977496:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 83628 low water 2055338 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.977513:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 82348 low water 2055978 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.981933:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 82348 low water 2055978 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.981963:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 81068 low water 2056618 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.983539:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 400660 low water 1896822 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.983558:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 400660 low water 1896822 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.983570:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 400660 low water 1896822 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.983575:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 400660 low water 1896822 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.984500:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 531052 low water 1831626 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.984518:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 531052 low water 1831626 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.984523:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 531052 low water 1831626 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.985389:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 531052 low water 1831626 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.985407:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 531052 low water 1831626 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.986053:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 531052 low water 1831626 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.986069:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 531052 low water 1831626 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.986080:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 531052 low water 1831626 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.990482:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.990495:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.990500:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:0.1:1786173175.991427:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:0.1:1786173175.991443:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.992556:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.992578:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.993652:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.993672:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.993963:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.993976:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.994246:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.994261:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.994557:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.994572:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.995010:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 807172 low water 1693566 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173175.995032:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 805892 low water 1694206 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.026428:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 933724 low water 1630290 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.026447:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 933724 low water 1630290 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.026471:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 933724 low water 1630290 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.026477:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 933724 low water 1630290 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.031668:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 933724 low water 1630290 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.035735:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.035758:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.035768:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.037218:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.037237:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.038334:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.038366:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.038379:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.039452:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.039591:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.040360:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.040377:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1380720 low water 1406792 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173176.041401:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1444636 low water 1374834 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:3.1:1786173176.649508:0:188491:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342600. 00010000:00000010:3.1:1786173183.221852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974000. 00010000:00000010:3.1:1786173183.288481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786173186.685659:0:2385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00000800:00000200:3.1:1786173190.662071:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -24164 low water 2109234 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.662093:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -25444 low water 2109874 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.663483:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -25444 low water 2109874 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.663510:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -26724 low water 2110514 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666162:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -6268 low water 2100286 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666201:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -7548 low water 2100926 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666216:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -7548 low water 2100926 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666249:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 11628 low water 2091338 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666262:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 11628 low water 2091338 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666267:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 11628 low water 2091338 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666289:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 48700 low water 2072802 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666305:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 103668 low water 2045318 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666321:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 158636 low water 2017834 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666339:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 213604 low water 1990350 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666355:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 223832 low water 1985236 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666368:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 278800 low water 1957752 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666392:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 289028 low water 1952638 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666398:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 289028 low water 1952638 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666414:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 343996 low water 1925154 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666453:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 354224 low water 1920040 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.666481:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 409192 low water 1892556 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.672066:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 419420 low water 1887442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.672095:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 419420 low water 1887442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.672107:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 419420 low water 1887442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.672113:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 419420 low water 1887442 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.672150:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 474388 low water 1859958 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.672156:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 474388 low water 1859958 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.673436:0:1348:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 474388 low water 1859958 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.673458:0:1348:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 474388 low water 1859958 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.673470:0:1348:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 474388 low water 1859958 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.673480:0:1348:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 474388 low water 1859958 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.682549:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 484616 low water 1854844 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.682579:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 484616 low water 1854844 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.682591:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 484616 low water 1854844 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.682597:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 484616 low water 1854844 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.698806:0:188491:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.699809:0:188491:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.699857:0:188491:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.699873:0:188491:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.699886:0:188491:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.699897:0:188491:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.699907:0:188491:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1254168 low water 1470068 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.699991:0:188491:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1318084 low water 1438110 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173190.700018:0:188491:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1445916 low water 1374194 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:0.1:1786173275.234503:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230400. 00010000:00000010:0.1:1786173275.234519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:3.1:1786173301.697481:0:188511:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:0.1:1786173305.325574:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9c00. 00010000:00000010:0.1:1786173305.325585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:3.1:1786173305.352678:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343c00. 00000080:00000010:0.1:1786173305.371466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca1c0. 00010000:00000010:0.1:1786173305.875866:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786173308.920746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f970800. 00010000:00000010:3.1:1786173312.350312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00000800:00000200:3.1:1786173315.170385:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.170405:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.170418:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.170430:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.170442:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace -36524 low water 2115414 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.172832:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.172864:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.172878:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.172890:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.172900:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.172907:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.173552:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.173568:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.173578:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.173588:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.173597:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 231928 low water 1981188 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.177009:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.177026:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.177037:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.177049:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.177058:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.177063:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 295844 low water 1949230 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.177103:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.180288:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.180305:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.180315:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.180324:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.180333:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 871088 low water 1661608 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173315.184008:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1509832 low water 1342236 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:3.1:1786173348.871444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:3.1:1786173348.875814:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10db1800. 00000800:00000200:3.1:1786173349.542576:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 779772 low water 1707266 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.542596:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 779772 low water 1707266 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.542607:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 779772 low water 1707266 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.542616:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 779772 low water 1707266 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.542627:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 779772 low water 1707266 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.542633:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 779772 low water 1707266 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.543121:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 779772 low water 1707266 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.543137:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 779772 low water 1707266 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.543148:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 779772 low water 1707266 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.543159:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 779772 low water 1707266 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.543170:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 779772 low water 1707266 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.543339:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 907604 low water 1643350 conn ffff9b9e03cc4000 blocked idle queued 00000800:00000200:3.1:1786173349.543892:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9b9e15862a00 wspace 1802428 low water 1195938 conn ffff9b9e03cc4000 blocked idle queued 00010000:00000010:3.1:1786173357.746933:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa000. 00010000:00000010:3.1:1786173357.746944:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343200. 00010000:00000010:0.1:1786173357.752525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230100. 00010000:00000010:0.1:1786173357.752536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122cc00. 00010000:00000010:3.1:1786173358.590268:0:188533:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa000. 00010000:00000010:3.1:1786173358.590277:0:188533:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343200. 00010000:00000010:3.1:1786173358.683411:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3f00. 00010000:00000010:3.1:1786173358.683433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1089ae00. 00010000:00000010:0.1:1786173360.460413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:3.1:1786173364.285314:0:188535:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5e00. 00010000:00000010:3.1:1786173364.319635:0:188342:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343200. 00000080:00000010:0.1:1786173365.434421:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02172100. 00010000:00000010:3.1:1786173365.480932:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342400. 00010000:00000010:3.1:1786173365.480948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343800. 00010000:00000010:3.1:1786173365.480952:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974a00. 00010000:00000010:3.1:1786173365.480956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10db1200. 00010000:00000010:3.1:1786173365.480961:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9d00. 00010000:00000010:3.1:1786173365.480965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:3.1:1786173365.480969:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3300. 00010000:00000010:3.1:1786173365.480973:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00000080:00000010:3.1:1786173365.530466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e3117cca0. 00000080:00000010:3.1:1786173366.249443:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02172940. 00010000:00000010:0.1:1786173369.935442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974800. 00010000:00000010:0.1:1786173369.935449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230a00. 00010000:00000010:0.1:1786173369.935453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943600. 00010000:00000010:0.1:1786173369.935456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230200. 00010000:00000010:0.1:1786173369.935459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786173369.935463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3900. 00010000:00000010:0.1:1786173369.935466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:3.1:1786173369.940554:0:188756:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f806e00. 00010000:00000010:0.1:1786173369.946668:0:188733:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3400. 00010000:00000010:0.1:1786173369.946692:0:188733:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5a00. 00010000:00000010:3.1:1786173375.426435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa500. 00010000:00000010:3.1:1786173375.426459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342000. 00010000:00000010:3.1:1786173375.441470:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aab00. 00010000:00000010:3.1:1786173375.477504:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa500. 00010000:00000010:3.1:1786173375.477521:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343000. 00000080:00000010:0.1:1786173375.520595:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca040. 00010000:00000010:3.1:1786173375.620543:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230500. 00010000:00000010:3.1:1786173375.620556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807c00. 00010000:00000010:0.1:1786173375.714559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807800. 00010000:00000010:3.1:1786173439.950409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342e00. 00010000:00000010:3.1:1786173439.950429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3400. 00010000:00000010:3.1:1786173439.950434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:3.1:1786173440.003921:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaf00. 00010000:00000010:3.1:1786173440.019407:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaa00. 00010000:00000010:3.1:1786173440.019421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343a00. 00010000:00000010:3.1:1786173440.056509:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa600. 00010000:00000010:3.1:1786173440.056520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342e00. 00010000:00000010:3.1:1786173440.287400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9100. 00010000:00000010:3.1:1786173440.287424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5800. 00010000:00000010:3.1:1786173440.471424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807200. 00010000:00000010:0.1:1786173440.491104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5a00. 00010000:00000010:3.1:1786173440.505411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807400. 00000080:00000010:0.1:1786173440.551035:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e093232e0. 00010000:00000010:3.1:1786173440.576506:0:189723:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230e00. 00010000:00000010:3.1:1786173440.576520:0:189723:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807600. 00010000:00000010:3.1:1786173450.253526:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3500. 00010000:00000010:3.1:1786173450.253551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:3.1:1786173450.264532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:3.1:1786173450.286510:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:3.1:1786173450.286525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343400. 00010000:00000010:3.1:1786173450.362406:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:3.1:1786173450.362424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343400. 00010000:00000010:3.1:1786173450.362427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:0.1:1786173450.376638:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342c00. 00000080:00000010:0.1:1786173450.429499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca100. 00010000:00000010:0.1:1786173450.449068:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230e00. 00010000:00000010:0.1:1786173450.449085:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807a00. 00000080:00000010:0.1:1786173450.474524:0:190318:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbfd60. 00010000:00000010:3.1:1786173458.050181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aae00. 00010000:00000010:3.1:1786173458.050197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342c00. 00010000:00000010:3.1:1786173458.071412:0:178995:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa300. 00010000:00000010:3.1:1786173458.077799:0:2395:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aae00. 00010000:00000010:3.1:1786173458.077806:0:2395:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342200. 00010000:00000010:3.1:1786173458.091393:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:3.1:1786173458.091409:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342c00. 00010000:00000010:3.1:1786173458.243512:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9d00. 00010000:00000010:3.1:1786173458.243519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4800. 00000080:00000010:0.1:1786173563.663882:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02170000. 00010000:00000010:0.1:1786173564.482826:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9200. 00010000:00000010:0.1:1786173564.482837:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4c00. 00010000:00000010:3.1:1786173582.751546:0:192215:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1c00. 00010000:00000010:3.1:1786173582.751559:0:192215:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc0600. 00010000:00000010:3.1:1786173582.751564:0:192215:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3b00. 00010000:00000010:3.1:1786173582.751570:0:192215:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1e00. 00010000:00000010:3.1:1786173582.751572:0:192215:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3900. 00010000:00000010:3.1:1786173582.751574:0:192215:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc0800. 00000080:00000010:3.1:1786173582.789603:0:192215:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e39fbfbe0. 00000080:00000010:3.1:1786173606.160522:0:192220:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02176300. 00000080:00000010:0.1:1786173806.655394:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e095ca940. 00000080:00000010:0.1:1786173806.655423:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020df380. 00000080:00000010:0.1:1786173806.655429:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020da940. 00000080:00000010:0.1:1786173806.655437:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020d9080. 00000080:00000010:0.1:1786173806.655442:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020dd280. 00000080:00000010:0.1:1786173806.655451:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020dca40. 00000080:00000010:0.1:1786173806.655457:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020dc200. 00000080:00000010:0.1:1786173806.655462:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119fe300. 00000080:00000010:0.1:1786173806.655467:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119fb180. 00000080:00000010:0.1:1786173806.655472:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119f98c0. 00000080:00000010:0.1:1786173806.655479:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020deb40. 00000080:00000010:0.1:1786173806.655484:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020d8840. 00000080:00000010:0.1:1786173806.655490:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020d8000. 00000080:00000010:0.1:1786173806.655495:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020de300. 00000080:00000010:0.1:1786173806.655500:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020db180. 00000080:00000010:0.1:1786173806.655505:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020d98c0. 00000080:00000010:0.1:1786173806.655511:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020db9c0. 00000080:00000010:0.1:1786173806.655517:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f2940. 00000080:00000010:0.1:1786173806.655567:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f1080. 00000080:00000010:0.1:1786173806.655581:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f5280. 00000080:00000010:0.1:1786173806.655589:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f4a40. 00000080:00000010:0.1:1786173806.655598:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f4200. 00000080:00000010:0.1:1786173806.655603:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f6b40. 00000080:00000010:0.1:1786173806.655610:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f0840. 00000080:00000010:0.1:1786173806.655615:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f0000. 00000080:00000010:0.1:1786173806.662389:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f6300. 00000080:00000010:0.1:1786173806.662407:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119fb9c0. 00000080:00000010:0.1:1786173806.662415:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119fdac0. 00000080:00000010:0.1:1786173806.662424:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119fa100. 00000080:00000010:0.1:1786173806.662431:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f3180. 00000080:00000010:0.1:1786173806.662436:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f18c0. 00000080:00000010:0.1:1786173806.662444:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f39c0. 00000080:00000010:0.1:1786173806.662450:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f5ac0. 00000080:00000010:0.1:1786173806.662453:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f2100. 00000080:00000010:0.1:1786173806.662459:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119ff380. 00000080:00000010:0.1:1786173806.662462:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119fa940. 00000080:00000010:0.1:1786173806.662465:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119f9080. 00000080:00000010:0.1:1786173806.662469:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e099f7380. 00000080:00000010:0.1:1786173806.662473:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09686300. 00000080:00000010:0.1:1786173806.662476:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09683180. 00000080:00000010:0.1:1786173806.662482:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e096818c0. 00000080:00000010:0.1:1786173806.662486:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e096839c0. 00000080:00000010:0.1:1786173806.662490:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09685ac0. 00000080:00000010:0.1:1786173806.662494:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09682100. 00000080:00000010:0.1:1786173806.667403:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f48000. 00000080:00000010:0.1:1786173806.667419:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e022218c0. 00000080:00000010:0.1:1786173806.667435:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09687380. 00000080:00000010:0.1:1786173806.667442:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09682940. 00000080:00000010:0.1:1786173806.667450:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09681080. 00000080:00000010:0.1:1786173806.667455:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09685280. 00000080:00000010:0.1:1786173806.667471:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09684a40. 00000080:00000010:0.1:1786173806.667475:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09684200. 00000080:00000010:0.1:1786173806.667480:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f4e300. 00000080:00000010:0.1:1786173806.667486:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f4b180. 00000080:00000010:0.1:1786173806.667490:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f498c0. 00000080:00000010:0.1:1786173806.667496:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f4b9c0. 00000080:00000010:0.1:1786173806.667501:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f4dac0. 00000080:00000010:0.1:1786173806.667505:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f4a100. 00000080:00000010:0.1:1786173806.667510:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f4f380. 00000080:00000010:0.1:1786173806.667515:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09686b40. 00000080:00000010:0.1:1786173806.667519:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09680840. 00000080:00000010:0.1:1786173806.667522:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09680000. 00000080:00000010:0.1:1786173806.667526:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c4200. 00000080:00000010:0.1:1786173806.667532:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094498c0. 00000080:00000010:0.1:1786173806.667538:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0944b9c0. 00000080:00000010:0.1:1786173806.667543:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0944dac0. 00000080:00000010:0.1:1786173806.667548:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0944a100. 00000080:00000010:0.1:1786173806.667552:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0944f380. 00000080:00000010:0.1:1786173806.667556:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0944a940. 00000080:00000010:0.1:1786173806.667560:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09449080. 00000080:00000010:0.1:1786173806.667564:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0944d280. 00000080:00000010:0.1:1786173806.667567:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0944ca40. 00000080:00000010:0.1:1786173806.667572:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c6b40. 00000080:00000010:0.1:1786173806.667577:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c0840. 00000080:00000010:0.1:1786173806.667581:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c0000. 00000080:00000010:0.1:1786173806.667585:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c6300. 00000080:00000010:0.1:1786173806.667589:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c3180. 00000080:00000010:0.1:1786173806.667593:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c18c0. 00000080:00000010:0.1:1786173806.667598:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c39c0. 00000080:00000010:0.1:1786173806.667602:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c5ac0. 00000080:00000010:0.1:1786173806.667606:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c2100. 00000080:00000010:0.1:1786173806.667616:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c7380. 00000080:00000010:0.1:1786173806.667621:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c2940. 00000080:00000010:0.1:1786173806.667629:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c1080. 00000080:00000010:0.1:1786173806.667634:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c5280. 00000080:00000010:0.1:1786173806.667638:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c4a40. 00000080:00000010:0.1:1786173806.667643:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e06300. 00000080:00000010:0.1:1786173806.667648:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e03180. 00000080:00000010:0.1:1786173806.667654:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e018c0. 00000080:00000010:0.1:1786173806.667658:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e039c0. 00000080:00000010:0.1:1786173806.667663:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e05ac0. 00000080:00000010:0.1:1786173806.667667:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e02100. 00000080:00000010:0.1:1786173806.667671:0:201118:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e07380. 00000080:00000010:0.1:1786173806.667698:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e02940. 00000080:00000010:0.1:1786173806.667704:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e01080. 00000080:00000010:0.1:1786173806.667710:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e05280. 00000080:00000010:0.1:1786173806.667716:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e04a40. 00000080:00000010:0.1:1786173806.667724:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e04200. 00000080:00000010:0.1:1786173806.667733:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e06b40. 00000080:00000010:0.1:1786173806.667738:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e00840. 00000080:00000010:0.1:1786173806.667742:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e00000. 00000080:00000010:0.1:1786173806.667751:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e998c0. 00000080:00000010:0.1:1786173806.667757:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9b9c0. 00000080:00000010:0.1:1786173806.667770:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9dac0. 00000080:00000010:0.1:1786173806.667776:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9a100. 00000080:00000010:0.1:1786173806.667783:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9f380. 00000080:00000010:0.1:1786173806.667788:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9a940. 00000080:00000010:0.1:1786173806.667792:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e99080. 00000080:00000010:0.1:1786173806.667797:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f4a940. 00000080:00000010:0.1:1786173806.667801:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f49080. 00000080:00000010:0.1:1786173806.667805:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f4d280. 00000080:00000010:0.1:1786173806.667809:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f4ca40. 00000080:00000010:0.1:1786173806.667820:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f4c200. 00000080:00000010:0.1:1786173806.667825:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f4eb40. 00000080:00000010:0.1:1786173806.667829:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f48840. 00000080:00000010:0.1:1786173806.667833:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c66b40. 00000080:00000010:0.1:1786173806.667837:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c60840. 00000080:00000010:0.1:1786173806.667841:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c60000. 00000080:00000010:0.1:1786173806.667845:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c66300. 00000080:00000010:0.1:1786173806.667849:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9d280. 00000080:00000010:0.1:1786173806.667853:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9ca40. 00000080:00000010:0.1:1786173806.667859:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9c200. 00000080:00000010:0.1:1786173806.667863:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9eb40. 00000080:00000010:0.1:1786173806.667868:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e98840. 00000080:00000010:0.1:1786173806.667872:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e98000. 00000080:00000010:0.1:1786173806.667875:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9e300. 00000080:00000010:0.1:1786173806.667879:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9b180. 00000080:00000010:0.1:1786173806.667884:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094bca40. 00000080:00000010:0.1:1786173806.667918:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094bc200. 00000080:00000010:0.1:1786173806.667924:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094beb40. 00000080:00000010:0.1:1786173806.667929:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094b8840. 00000080:00000010:0.1:1786173806.667933:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094b8000. 00000080:00000010:0.1:1786173806.667936:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094be300. 00000080:00000010:0.1:1786173806.667941:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094bb180. 00000080:00000010:0.1:1786173806.667945:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094b98c0. 00000080:00000010:0.1:1786173806.667949:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094bb9c0. 00000080:00000010:0.1:1786173806.667952:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094bdac0. 00000080:00000010:0.1:1786173806.667957:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02224200. 00000080:00000010:0.1:1786173806.667962:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02226b40. 00000080:00000010:0.1:1786173806.667966:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02225ac0. 00000080:00000010:0.1:1786173806.667970:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02223180. 00000080:00000010:0.1:1786173806.667974:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02226300. 00000080:00000010:0.1:1786173806.667987:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c63180. 00000080:00000010:0.1:1786173806.667998:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09401080. 00000080:00000010:0.1:1786173806.668004:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09400000. 00000080:00000010:0.1:1786173806.668008:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09402940. 00000080:00000010:0.1:1786173806.668021:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0944c200. 00000080:00000010:0.1:1786173806.668025:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0944eb40. 00000080:00000010:0.1:1786173806.668031:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09448840. 00000080:00000010:0.1:1786173806.668035:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09448000. 00000080:00000010:0.1:1786173806.668039:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0944e300. 00000080:00000010:0.1:1786173806.668043:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0944b180. 00000080:00000010:0.1:1786173806.668053:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c2940. 00000080:00000010:0.1:1786173806.668059:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c1080. 00000080:00000010:0.1:1786173806.668064:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c5280. 00000080:00000010:0.1:1786173806.668068:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c4a40. 00000080:00000010:0.1:1786173806.668072:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c0000. 00000080:00000010:0.1:1786173806.668077:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c6300. 00000080:00000010:0.1:1786173806.668084:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094b9080. 00000080:00000010:0.1:1786173806.668088:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094bd280. 00000080:00000010:0.1:1786173806.668091:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094bf380. 00000080:00000010:0.1:1786173806.668095:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094ba940. 00000080:00000010:0.1:1786173806.668113:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0951d280. 00000080:00000010:0.1:1786173806.668120:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0951ca40. 00000080:00000010:0.1:1786173806.668124:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0951c200. 00000080:00000010:0.1:1786173806.668127:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0951eb40. 00000080:00000010:0.1:1786173806.668132:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09518840. 00000080:00000010:0.1:1786173806.668136:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09518000. 00000080:00000010:0.1:1786173806.668141:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0951e300. 00000080:00000010:0.1:1786173806.668145:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0951b180. 00000080:00000010:0.1:1786173806.668149:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c18c0. 00000080:00000010:0.1:1786173806.668153:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c39c0. 00000080:00000010:0.1:1786173806.668157:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c5ac0. 00000080:00000010:0.1:1786173806.668162:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c2100. 00000080:00000010:0.1:1786173806.668171:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c7380. 00000080:00000010:0.1:1786173806.668175:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c6b40. 00000080:00000010:0.1:1786173806.668179:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c0840. 00000080:00000010:0.1:1786173806.668182:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c4200. 00000080:00000010:0.1:1786173806.668188:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e095198c0. 00000080:00000010:0.1:1786173806.668192:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0951b9c0. 00000080:00000010:0.1:1786173806.668195:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0951dac0. 00000080:00000010:0.1:1786173806.668199:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0951a100. 00000080:00000010:0.1:1786173806.668203:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a3c3180. 00000080:00000010:0.1:1786173806.668208:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c639c0. 00000080:00000010:0.1:1786173806.668212:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15c65ac0. 00000080:00000010:0.1:1786173806.668222:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0951f380. 00000080:00000010:0.1:1786173806.668227:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e094ba100. 00000080:00000010:0.1:1786173806.668232:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09519080. 00000080:00000010:0.1:1786173806.668237:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e11918000. 00000080:00000010:0.1:1786173806.668241:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1191e300. 00000080:00000010:0.1:1786173806.668245:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1191b180. 00000080:00000010:0.1:1786173806.668248:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e0951a940. 00000080:00000010:0.1:1786173806.668253:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e119198c0. 00000080:00000010:0.1:1786173806.668258:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f72100. 00000080:00000010:0.1:1786173806.668262:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1191dac0. 00000080:00000010:0.1:1786173806.668269:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f77380. 00000080:00000010:0.1:1786173806.668273:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f72940. 00000080:00000010:0.1:1786173806.668277:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1191b9c0. 00000080:00000010:0.1:1786173806.668284:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1191a100. 00000080:00000010:0.1:1786173806.668287:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15f71080. 00000080:00000010:0.1:1786173806.668299:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1191f380. 00000080:00000010:0.1:1786173806.668303:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e1191a940. 00000080:00000010:0.1:1786173806.668309:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d74200. 00000080:00000010:0.1:1786173806.668314:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32fe0000. 00000080:00000010:0.1:1786173806.668319:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e095939c0. 00000080:00000010:0.1:1786173806.668328:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e09595ac0. 00000080:00000010:0.1:1786173806.668332:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e095918c0. 00000080:00000010:0.1:1786173806.668337:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32fe6b40. 00000080:00000010:0.1:1786173806.668341:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32fe4200. 00000080:00000010:0.1:1786173806.668345:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32fe0840. 00000080:00000010:0.1:1786173806.668354:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a28dac0. 00000080:00000010:0.1:1786173806.668388:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02176b40. 00000080:00000010:0.1:1786173806.668405:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02175ac0. 00000080:00000010:0.1:1786173806.668431:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32d1d280. 00010000:00000010:3.1:1786173806.670399:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3900. 00010000:00000010:3.1:1786173806.670413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1800. 00000080:00000010:0.1:1786173806.670419:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32d1ca40. 00000080:00000010:0.1:1786173806.670443:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02172940. 00000080:00000010:0.1:1786173806.670465:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02174a40. 00000080:00000010:0.1:1786173806.670480:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02174200. 00000080:00000010:0.1:1786173806.670490:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e02170840. 00000080:00000010:3.1:1786173809.648415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e1c42a0. 00000080:00000010:3.1:1786173809.648426:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e32d18840. 00010000:00000010:0.1:1786173809.652392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342200. 00010000:00000010:0.1:1786173809.652403:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaa00. 00010000:00000010:0.1:1786173809.652415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343a00. 00010000:00000010:3.1:1786173809.670452:0:199499:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc4a00. 00010000:00000010:3.1:1786173809.670457:0:199499:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230600. 00010000:00000010:0.1:1786173809.683406:0:201785:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1e00. 00010000:00000010:0.1:1786173809.683422:0:201785:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3700. 00010000:00000010:3.1:1786173817.780399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343000. 00000080:00000010:3.1:1786173817.791415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca260. 00010000:00000010:3.1:1786173817.791442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230d00. 00010000:00000010:3.1:1786173817.791451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc4200. 00010000:00000010:0.1:1786173818.098389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1e00. 00000080:00000010:0.1:1786173818.098398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e312889c0. 00010000:00000010:0.1:1786173818.110400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc0200. 00000080:00000010:0.1:1786173818.110412:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288160. 00010000:00000010:3.1:1786173820.442448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5200. 00010000:00000010:3.1:1786173820.446427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342e00. 00000080:00000010:0.1:1786173820.454699:0:1011:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcae00. 00010000:00000010:0.1:1786173820.454716:0:1011:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230100. 00010000:00000010:0.1:1786173820.454728:0:1011:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc4c00. 00010000:00000010:0.1:1786173820.461602:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3800. 00010000:00000010:0.1:1786173820.461606:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3500. 00010000:00000010:0.1:1786173820.461610:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1a00. 00000080:00000010:3.1:1786173820.466429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ae0. 00000080:00000010:3.1:1786173820.519395:0:204062:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09323c00. 00000080:00000010:3.1:1786173820.519407:0:204062:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaf20. 00000080:00000010:3.1:1786173820.519415:0:204062:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288380. 00010000:00000010:3.1:1786173820.529409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342600. 00010000:00000010:3.1:1786173820.529422:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:3.1:1786173820.529429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343400. 00010000:00000010:3.1:1786173820.529436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa800. 00010000:00000010:3.1:1786173820.529439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342e00. 00010000:00000010:3.1:1786173820.529444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3c00. 00010000:00000010:3.1:1786173820.529454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1400. 00010000:00000010:3.1:1786173820.721384:0:204223:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa100. 00010000:00000010:3.1:1786173820.721395:0:204223:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343400. 00010000:00000010:3.1:1786173820.721404:0:204223:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa500. 00010000:00000010:3.1:1786173820.721410:0:204223:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342e00. 00000080:00000010:3.1:1786173820.730398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca340. 00010000:00000010:3.1:1786173820.733406:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa700. 00010000:00000010:3.1:1786173820.733421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343200. 00010000:00000010:3.1:1786173820.733427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa800. 00010000:00000010:3.1:1786173820.733430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342400. 00000080:00000010:3.1:1786173820.737386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca520. 00000080:00000010:3.1:1786173820.737465:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a28dac0. 00010000:00000010:0.1:1786173830.496398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1e00. 00000080:00000010:0.1:1786173830.511396:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288240. 00000080:00000010:0.1:1786173830.516385:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288700. 00010000:00000010:0.1:1786173830.520389:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1400. 00000080:00000010:0.1:1786173830.524398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288ca0. 00000080:00000010:0.1:1786173830.528384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288700. 00010000:00000010:0.1:1786173830.536385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1400. 00010000:00000010:0.1:1786173830.546380:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1400. 00000080:00000010:0.1:1786173830.554385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288ca0. 00000080:00000010:0.1:1786173830.558388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288820. 00010000:00000010:0.1:1786173830.566402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1600. 00010000:00000010:0.1:1786173830.574384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc0e00. 00000080:00000010:0.1:1786173830.579379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288700. 00000080:00000010:0.1:1786173830.583400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e312886e0. 00010000:00000010:0.1:1786173830.587373:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc0e00. 00000080:00000010:0.1:1786173830.596397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288d40. 00000080:00000010:0.1:1786173830.605376:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e312886e0. 00010000:00000010:0.1:1786173830.609390:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3a00. 00010000:00000010:0.1:1786173830.609397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1800. 00010000:00000010:0.1:1786173830.613386:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3800. 00010000:00000010:0.1:1786173830.616512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10560200. 00000080:00000010:0.1:1786173830.625415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e312883a0. 00000080:00000010:0.1:1786173830.630376:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e312881e0. 00010000:00000010:0.1:1786173830.640395:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3c00. 00010000:00000010:0.1:1786173830.640405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10560e00. 00010000:00000010:3.1:1786173830.649390:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:3.1:1786173830.661393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343a00. 00010000:00000010:3.1:1786173830.666393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343600. 00000080:00000010:3.1:1786173830.671375:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca540. 00000080:00000010:3.1:1786173830.677392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaa40. 00010000:00000010:3.1:1786173830.687377:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaa00. 00010000:00000010:3.1:1786173830.687381:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343c00. 00010000:00000010:3.1:1786173830.692391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa000. 00010000:00000010:3.1:1786173830.700378:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa200. 00010000:00000010:3.1:1786173830.700383:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342600. 00010000:00000010:3.1:1786173830.711414:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa000. 00010000:00000010:3.1:1786173830.715375:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aae00. 00010000:00000010:3.1:1786173830.715380:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342600. 00010000:00000010:3.1:1786173830.719415:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaf00. 00010000:00000010:3.1:1786173830.723449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343e00. 00000080:00000010:3.1:1786173830.728383:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca580. 00000080:00000010:3.1:1786173830.734405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca380. 00010000:00000010:3.1:1786173830.744391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa000. 00010000:00000010:3.1:1786173830.744400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342400. 00010000:00000010:3.1:1786173830.757388:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa400. 00010000:00000010:3.1:1786173830.757393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342400. 00010000:00000010:3.1:1786173830.770651:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa900. 00010000:00000010:3.1:1786173830.782405:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b400. 00010000:00000010:3.1:1786173830.782410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc3e00. 00010000:00000010:3.1:1786173830.790398:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68bd00. 00010000:00000010:3.1:1786173830.799405:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b100. 00010000:00000010:3.1:1786173830.799413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2800. 00010000:00000010:3.1:1786173830.805426:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b700. 00010000:00000010:3.1:1786173830.813432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b800. 00010000:00000010:3.1:1786173830.813440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2e00. 00010000:00000010:3.1:1786173830.821384:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b700. 00010000:00000010:3.1:1786173830.825383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2e00. 00000080:00000010:3.1:1786173830.829381:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca3c0. 00000080:00000010:3.1:1786173830.834403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcafe0. 00010000:00000010:3.1:1786173830.843382:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2e00. 00000080:00000010:3.1:1786173830.848375:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcae00. 00000080:00000010:3.1:1786173830.848384:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcafe0. 00010000:00000010:3.1:1786173830.857388:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b300. 00010000:00000010:3.1:1786173830.857393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc3800. 00010000:00000010:3.1:1786173830.865395:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b200. 00010000:00000010:2.1:1786173830.877400:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4c00. 00000080:00000010:2.1:1786173830.883390:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dae0. 00000080:00000010:2.1:1786173830.887419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d120. 00010000:00000010:2.1:1786173830.891410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4c00. 00000080:00000010:2.1:1786173830.900383:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d080. 00000080:00000010:2.1:1786173830.905394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d120. 00010000:00000010:2.1:1786173830.905404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4e00. 00000080:00000010:2.1:1786173830.915398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dc60. 00000080:00000010:2.1:1786173830.920390:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d080. 00010000:00000010:2.1:1786173830.928399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4600. 00000080:00000010:2.1:1786173830.933760:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d120. 00000080:00000010:2.1:1786173830.941401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d080. 00010000:00000010:2.1:1786173830.950393:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9700. 00010000:00000010:2.1:1786173830.950401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5a00. 00010000:00000010:2.1:1786173830.954391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9d00. 00010000:00000010:2.1:1786173830.958388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4800. 00010000:00000010:2.1:1786173830.980397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974000. 00000080:00000010:2.1:1786173830.984393:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d860. 00000080:00000010:2.1:1786173830.989397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dbe0. 00010000:00000010:2.1:1786173830.999401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418500. 00010000:00000010:2.1:1786173830.999408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00010000:00000010:2.1:1786173831.008385:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418c00. 00010000:00000010:2.1:1786173831.016394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975800. 00000080:00000010:2.1:1786173831.024382:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d040. 00000080:00000010:2.1:1786173831.031389:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dbe0. 00010000:00000010:2.1:1786173831.031401:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:2.1:1786173831.036377:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995800. 00000080:00000010:2.1:1786173831.036380:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dee0. 00000080:00000010:2.1:1786173831.041388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dbe0. 00010000:00000010:2.1:1786173831.057389:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418100. 00010000:00000010:2.1:1786173831.057396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995200. 00010000:00000010:2.1:1786173831.065615:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418a00. 00010000:00000010:2.1:1786173831.065638:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995600. 00000080:00000010:2.1:1786173831.069394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18de60. 00000080:00000010:2.1:1786173831.077386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dc00. 00010000:00000010:2.1:1786173831.094381:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418f00. 00010000:00000010:2.1:1786173831.094388:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31994e00. 00010000:00000010:2.1:1786173831.105385:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459600. 00010000:00000010:2.1:1786173831.116385:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459d00. 00010000:00000010:2.1:1786173831.116392:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995c00. 00010000:00000010:2.1:1786173831.121402:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459900. 00010000:00000010:2.1:1786173831.130387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459d00. 00010000:00000010:2.1:1786173831.130394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31994000. 00010000:00000010:2.1:1786173831.144379:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995000. 00010000:00000010:2.1:1786173831.149376:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31994400. 00000080:00000010:2.1:1786173831.153379:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d000. 00000080:00000010:2.1:1786173831.158413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d580. 00010000:00000010:2.1:1786173831.168382:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459800. 00010000:00000010:2.1:1786173831.168389:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:2.1:1786173831.179386:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459800. 00010000:00000010:2.1:1786173831.192372:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459c00. 00010000:00000010:2.1:1786173831.192377:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3a00. 00010000:00000010:2.1:1786173831.192384:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459800. 00010000:00000010:2.1:1786173831.198385:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418f00. 00010000:00000010:2.1:1786173831.198389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3600. 00010000:00000010:2.1:1786173831.208384:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418f00. 00010000:00000010:2.1:1786173831.220381:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd2e00. 00000080:00000010:2.1:1786173831.224453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d0e0. 00000080:00000010:2.1:1786173831.228393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d6c0. 00010000:00000010:2.1:1786173831.232377:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd2e00. 00000080:00000010:2.1:1786173831.240393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d8e0. 00000080:00000010:2.1:1786173831.245387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d6c0. 00010000:00000010:2.1:1786173831.258390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3c00. 00010000:00000010:2.1:1786173831.262388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3000. 00000080:00000010:2.1:1786173831.266383:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d8e0. 00000080:00000010:2.1:1786173831.271380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d920. 00010000:00000010:2.1:1786173831.279384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3000. 00010000:00000010:2.1:1786173831.286383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261acc00. 00000080:00000010:2.1:1786173831.295374:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d8e0. 00000080:00000010:2.1:1786173831.299399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18db60. 00010000:00000010:2.1:1786173831.303385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261acc00. 00000080:00000010:2.1:1786173831.307388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d8e0. 00000080:00000010:2.1:1786173831.312380:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dcc0. 00010000:00000010:2.1:1786173831.320391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261acc00. 00010000:00000010:2.1:1786173831.324374:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ad000. 00000080:00000010:2.1:1786173831.328383:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d8e0. 00000080:00000010:2.1:1786173831.335607:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18db60. 00010000:00000010:2.1:1786173831.348487:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd100. 00010000:00000010:2.1:1786173831.348496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261aca00. 00010000:00000010:2.1:1786173831.353408:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af800. 00010000:00000010:2.1:1786173831.373422:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af100. 00010000:00000010:2.1:1786173831.373438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ad200. 00010000:00000010:2.1:1786173831.385392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af700. 00010000:00000010:2.1:1786173831.394392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af000. 00010000:00000010:2.1:1786173831.394400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ad400. 00010000:00000010:2.1:1786173831.399396:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af700. 00010000:00000010:2.1:1786173831.407404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ad400. 00000080:00000010:2.1:1786173831.412392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d1a0. 00000080:00000010:2.1:1786173831.425401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d440. 00010000:00000010:2.1:1786173831.433384:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fada400. 00000080:00000010:2.1:1786173831.437388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d9c0. 00000080:00000010:2.1:1786173831.443402:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d4c0. 00010000:00000010:2.1:1786173831.450466:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af600. 00010000:00000010:2.1:1786173831.450473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadb200. 00010000:00000010:2.1:1786173831.463380:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328afb00. 00010000:00000010:2.1:1786173831.467380:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadb200. 00000080:00000010:2.1:1786173831.472386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d440. 00000080:00000010:2.1:1786173831.477392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dda0. 00010000:00000010:2.1:1786173831.485482:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadb200. 00000080:00000010:2.1:1786173831.494385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18de80. 00000080:00000010:2.1:1786173831.494397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dda0. 00010000:00000010:2.1:1786173831.502400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328afe00. 00010000:00000010:2.1:1786173831.502407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadbe00. 00010000:00000010:2.1:1786173831.512377:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b0f000. 00010000:00000010:2.1:1786173831.524386:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328afe00. 00010000:00000010:2.1:1786173831.524391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171ca00. 00010000:00000010:2.1:1786173831.528372:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b0f000. 00010000:00000010:2.1:1786173831.532379:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171d800. 00010000:00000010:2.1:1786173831.540375:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171d200. 00000080:00000010:2.1:1786173831.544391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d500. 00000080:00000010:2.1:1786173831.548382:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d020. 00010000:00000010:2.1:1786173831.556371:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e138abb00. 00010000:00000010:2.1:1786173831.556375:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171c200. 00010000:00000010:2.1:1786173831.564384:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b0f000. 00010000:00000010:2.1:1786173831.573393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171c200. 00000080:00000010:2.1:1786173831.577395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dd00. 00000080:00000010:2.1:1786173831.586375:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dce0. 00010000:00000010:2.1:1786173831.593404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03000. 00010000:00000010:2.1:1786173831.593412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171de00. 00010000:00000010:2.1:1786173831.601403:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03e00. 00010000:00000010:3.1:1786173831.605383:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2a00. 00000080:00000010:3.1:1786173831.609387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaea0. 00000080:00000010:3.1:1786173831.613377:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca640. 00010000:00000010:3.1:1786173831.617379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2a00. 00000080:00000010:3.1:1786173831.621388:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca8e0. 00000080:00000010:3.1:1786173831.626394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaea0. 00010000:00000010:3.1:1786173831.635380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9e00. 00000080:00000010:3.1:1786173831.639383:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca8e0. 00000080:00000010:3.1:1786173831.643383:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca640. 00010000:00000010:3.1:1786173831.651384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9e00. 00010000:00000010:3.1:1786173831.657381:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9200. 00000080:00000010:3.1:1786173831.665388:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaea0. 00000080:00000010:3.1:1786173831.670420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca600. 00010000:00000010:3.1:1786173831.674404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e900. 00010000:00000010:3.1:1786173831.674409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb8200. 00010000:00000010:3.1:1786173831.684388:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e900. 00010000:00000010:3.1:1786173831.691380:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e500. 00010000:00000010:3.1:1786173831.691384:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb8e00. 00010000:00000010:3.1:1786173831.696388:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e900. 00010000:00000010:3.1:1786173831.705391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e500. 00010000:00000010:3.1:1786173831.705396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9a00. 00010000:00000010:3.1:1786173831.710381:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7ed00. 00010000:00000010:3.1:1786173831.719404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e600. 00010000:00000010:3.1:1786173831.719409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2a00. 00010000:00000010:3.1:1786173831.724380:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7ed00. 00010000:00000010:3.1:1786173831.732372:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45400. 00000080:00000010:3.1:1786173831.736372:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca360. 00000080:00000010:3.1:1786173831.740373:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca4c0. 00010000:00000010:3.1:1786173831.744383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45400. 00000080:00000010:3.1:1786173831.748393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca360. 00000080:00000010:3.1:1786173831.753392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca320. 00010000:00000010:3.1:1786173831.763377:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7ec00. 00010000:00000010:3.1:1786173831.763384:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45c00. 00010000:00000010:3.1:1786173831.772413:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b000. 00010000:00000010:3.1:1786173831.780377:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5400. 00010000:00000010:3.1:1786173831.780382:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45a00. 00010000:00000010:3.1:1786173831.788377:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5800. 00010000:00000010:3.1:1786173831.792381:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44400. 00000080:00000010:3.1:1786173831.800382:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca6c0. 00000080:00000010:3.1:1786173831.805390:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca7e0. 00010000:00000010:3.1:1786173831.810377:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44400. 00000080:00000010:3.1:1786173831.818373:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca7c0. 00000080:00000010:3.1:1786173831.822373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca7e0. 00010000:00000010:3.1:1786173831.830380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44000. 00010000:00000010:3.1:1786173831.834379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942200. 00000080:00000010:3.1:1786173831.838380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca7c0. 00000080:00000010:3.1:1786173831.842373:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca660. 00010000:00000010:3.1:1786173831.846404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942c00. 00010000:00000010:3.1:1786173831.850390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943600. 00000080:00000010:3.1:1786173831.850395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcabe0. 00000080:00000010:3.1:1786173831.854377:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca1c0. 00010000:00000010:3.1:1786173831.862377:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5600. 00010000:00000010:3.1:1786173831.862381:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943600. 00010000:00000010:3.1:1786173831.866370:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5900. 00010000:00000010:3.1:1786173831.870387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00000080:00000010:3.1:1786173831.874375:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca660. 00000080:00000010:3.1:1786173831.878382:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcabe0. 00010000:00000010:3.1:1786173831.878392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943600. 00000080:00000010:3.1:1786173831.883376:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca440. 00000080:00000010:3.1:1786173831.888380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcabe0. 00010000:00000010:3.1:1786173831.894396:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5700. 00010000:00000010:3.1:1786173831.894401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943c00. 00010000:00000010:3.1:1786173831.904371:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5700. 00010000:00000010:3.1:1786173831.912382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942a00. 00010000:00000010:3.1:1786173831.917397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943000. 00000080:00000010:3.1:1786173831.917403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca660. 00000080:00000010:3.1:1786173831.922392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaa00. 00010000:00000010:3.1:1786173831.931381:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac600. 00010000:00000010:3.1:1786173831.931386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44000. 00010000:00000010:3.1:1786173831.940393:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac800. 00010000:00000010:3.1:1786173831.949378:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac700. 00010000:00000010:3.1:1786173831.949383:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807600. 00010000:00000010:3.1:1786173831.957382:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dacb00. 00010000:00000010:3.1:1786173831.965389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807600. 00000080:00000010:3.1:1786173831.992465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca480. 00000080:00000010:3.1:1786173831.998391:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca280. 00010000:00000010:3.1:1786173832.008377:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac700. 00010000:00000010:3.1:1786173832.008381:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f806400. 00010000:00000010:3.1:1786173832.013382:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac400. 00010000:00000010:3.1:1786173832.019391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5700. 00010000:00000010:3.1:1786173832.019398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f806400. 00010000:00000010:3.1:1786173832.031376:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807800. 00000080:00000010:3.1:1786173832.035376:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca4a0. 00000080:00000010:3.1:1786173832.043376:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcacc0. 00010000:00000010:3.1:1786173832.047380:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807800. 00000080:00000010:3.1:1786173832.051392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca5a0. 00000080:00000010:3.1:1786173832.055393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcace0. 00010000:00000010:3.1:1786173832.060383:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2c00. 00010000:00000010:3.1:1786173832.060388:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44000. 00010000:00000010:3.1:1786173832.065382:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2000. 00010000:00000010:3.1:1786173832.075380:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00000080:00000010:3.1:1786173832.080378:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcacc0. 00000080:00000010:3.1:1786173832.092386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca1e0. 00010000:00000010:3.1:1786173832.100401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:3.1:1786173832.104370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00000080:00000010:3.1:1786173832.108378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcacc0. 00000080:00000010:3.1:1786173832.116377:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcae40. 00010000:00000010:3.1:1786173832.125387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2400. 00010000:00000010:3.1:1786173832.125394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786173832.125403:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2b00. 00010000:00000010:3.1:1786173832.130422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:3.1:1786173832.134398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00000080:00000010:3.1:1786173832.138372:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca220. 00000080:00000010:3.1:1786173832.142400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcae80. 00010000:00000010:2.1:1786173832.151397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc5600. 00000080:00000010:2.1:1786173832.205421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18df20. 00000080:00000010:2.1:1786173832.209384:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d240. 00010000:00000010:2.1:1786173832.218377:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03900. 00010000:00000010:2.1:1786173832.218384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171d000. 00010000:00000010:2.1:1786173832.227390:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03600. 00010000:00000010:2.1:1786173832.227395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0800. 00010000:00000010:2.1:1786173832.232394:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03200. 00010000:00000010:2.1:1786173832.246394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0800. 00000080:00000010:2.1:1786173832.250407:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d900. 00000080:00000010:2.1:1786173832.256397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d380. 00010000:00000010:2.1:1786173832.265384:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0800. 00000080:00000010:2.1:1786173832.270392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d900. 00000080:00000010:2.1:1786173832.279392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18de20. 00010000:00000010:2.1:1786173832.293375:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03700. 00010000:00000010:2.1:1786173832.293379:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0a00. 00010000:00000010:2.1:1786173832.302411:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03c00. 00010000:00000010:2.1:1786173832.316399:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03300. 00010000:00000010:2.1:1786173832.316415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0a00. 00010000:00000010:2.1:1786173832.320392:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03c00. 00010000:00000010:2.1:1786173832.324395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c1200. 00000080:00000010:2.1:1786173832.332397:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d520. 00000080:00000010:2.1:1786173832.337382:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ce0. 00010000:00000010:2.1:1786173832.346377:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03d00. 00010000:00000010:2.1:1786173832.346384:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c1600. 00010000:00000010:2.1:1786173832.359585:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f000. 00010000:00000010:2.1:1786173832.367384:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f800. 00010000:00000010:2.1:1786173832.367389:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171d000. 00010000:00000010:2.1:1786173832.376377:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818fe00. 00010000:00000010:2.1:1786173832.392380:0:205563:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f400. 00010000:00000010:2.1:1786173832.392386:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e552e00. 00010000:00000010:2.1:1786173832.416384:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f400. 00010000:00000010:2.1:1786173832.424376:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e553e00. 00000080:00000010:2.1:1786173832.428389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069780. 00000080:00000010:2.1:1786173832.428401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069520. 00010000:00000010:2.1:1786173832.436411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e552600. 00010000:00000010:2.1:1786173832.444394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e553000. 00000080:00000010:2.1:1786173832.444401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069780. 00000080:00000010:2.1:1786173832.448392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069920. 00010000:00000010:2.1:1786173832.456392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e552a00. 00010000:00000010:2.1:1786173832.466376:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e553800. 00000080:00000010:2.1:1786173832.474393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069780. 00000080:00000010:2.1:1786173832.474405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069d00. 00010000:00000010:2.1:1786173832.482387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e553800. 00000080:00000010:2.1:1786173832.486384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069820. 00000080:00000010:2.1:1786173832.490380:0:205563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069d00. 00010000:00000010:2.1:1786173832.500396:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e700. 00010000:00000010:2.1:1786173832.500406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373200. 00010000:00000010:2.1:1786173832.504388:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720ed00. 00010000:00000010:2.1:1786173832.508386:0:205563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373600. 00010000:00000010:2.1:1786173832.513393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372e00. 00000080:00000010:2.1:1786173832.517379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069780. 00000080:00000010:2.1:1786173832.521380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069be0. 00010000:00000010:2.1:1786173832.529390:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e000. 00010000:00000010:2.1:1786173832.529395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00010000:00000010:2.1:1786173832.533394:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e300. 00010000:00000010:3.1:1786173833.938387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc5800. 00010000:00000010:0.1:1786173833.938388:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9600. 00010000:00000010:0.1:1786173833.938400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5c00. 00010000:00000010:3.1:1786173836.655408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc5600. 00010000:00000010:3.1:1786173836.659394:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230e00. 00010000:00000010:3.1:1786173836.659403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc5400. 00010000:00000010:2.1:1786173838.090400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.094402:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069900. 00010000:00000010:2.1:1786173838.094416:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e300. 00010000:00000010:2.1:1786173838.094424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372a00. 00010000:00000010:3.1:1786173838.118385:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2e00. 00010000:00000010:3.1:1786173838.118391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031000. 00010000:00000010:3.1:1786173838.122387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2700. 00010000:00000010:3.1:1786173838.122392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031600. 00000080:00000010:3.1:1786173838.122395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca780. 00010000:00000010:3.1:1786173838.122403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00000080:00000010:3.1:1786173838.134774:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca7a0. 00010000:00000010:3.1:1786173838.150374:0:205949:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2400. 00010000:00000010:3.1:1786173838.150379:0:205949:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122de00. 00010000:00000010:3.1:1786173838.158389:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2300. 00010000:00000010:3.1:1786173838.162411:0:205950:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2700. 00010000:00000010:3.1:1786173838.162416:0:205950:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00000080:00000010:3.1:1786173838.172384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca7a0. 00010000:00000010:3.1:1786173838.176409:0:205951:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb400. 00010000:00000010:3.1:1786173838.176414:0:205951:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00000080:00000010:3.1:1786173838.176417:0:205951:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca160. 00010000:00000010:3.1:1786173838.193400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb400. 00010000:00000010:3.1:1786173838.193410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00000080:00000010:3.1:1786173838.193414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca160. 00010000:00000010:3.1:1786173838.201383:0:205953:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb300. 00010000:00000010:3.1:1786173838.201394:0:205953:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00000080:00000010:3.1:1786173838.201402:0:205953:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaae0. 00010000:00000010:3.1:1786173838.214392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb300. 00010000:00000010:3.1:1786173838.214397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00000080:00000010:3.1:1786173838.214400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaae0. 00010000:00000010:3.1:1786173838.226421:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb800. 00010000:00000010:3.1:1786173838.226444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb300. 00010000:00000010:3.1:1786173838.226458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00010000:00000010:2.1:1786173838.273382:0:205957:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8e00. 00010000:00000010:2.1:1786173838.273391:0:205957:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10560a00. 00000080:00000010:2.1:1786173838.273394:0:205957:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288840. 00010000:00000010:2.1:1786173838.289388:0:205958:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e300. 00010000:00000010:2.1:1786173838.289393:0:205958:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372a00. 00000080:00000010:2.1:1786173838.289400:0:205958:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069fc0. 00010000:00000010:2.1:1786173838.298669:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e900. 00010000:00000010:2.1:1786173838.298677:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.298681:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e110692a0. 00010000:00000010:3.1:1786173838.309388:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb100. 00010000:00000010:3.1:1786173838.309398:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebc00. 00010000:00000010:3.1:1786173838.309401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00010000:00000010:2.1:1786173838.371391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e900. 00010000:00000010:2.1:1786173838.371402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.371412:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069aa0. 00010000:00000010:3.1:1786173838.379389:0:205964:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebc00. 00010000:00000010:3.1:1786173838.383392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb100. 00010000:00000010:3.1:1786173838.383399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00000080:00000010:3.1:1786173838.389393:0:205965:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca7a0. 00010000:00000010:3.1:1786173838.401391:0:205966:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb100. 00010000:00000010:3.1:1786173838.401398:0:205966:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00000080:00000010:3.1:1786173838.401401:0:205966:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca7a0. 00010000:00000010:3.1:1786173838.421379:0:205968:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb100. 00010000:00000010:3.1:1786173838.421384:0:205968:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00000080:00000010:3.1:1786173838.421391:0:205968:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca7a0. 00010000:00000010:3.1:1786173838.426897:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebc00. 00010000:00000010:3.1:1786173838.426902:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00000080:00000010:3.1:1786173838.426905:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca020. 00010000:00000010:3.1:1786173838.438390:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebc00. 00010000:00000010:3.1:1786173838.438397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00000080:00000010:3.1:1786173838.438399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaa60. 00010000:00000010:2.1:1786173838.450399:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e900. 00010000:00000010:2.1:1786173838.450407:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e300. 00010000:00000010:2.1:1786173838.450411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00010000:00000010:2.1:1786173838.497393:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8b00. 00010000:00000010:2.1:1786173838.497400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10560400. 00000080:00000010:2.1:1786173838.497407:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e312884e0. 00010000:00000010:2.1:1786173838.501616:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e300. 00010000:00000010:2.1:1786173838.501620:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.501623:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069b20. 00010000:00000010:2.1:1786173838.514424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e300. 00010000:00000010:2.1:1786173838.518385:0:205975:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e900. 00010000:00000010:2.1:1786173838.518396:0:205975:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:3.1:1786173838.522594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069b20. 00010000:00000010:2.1:1786173838.534388:0:205976:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e900. 00010000:00000010:2.1:1786173838.534398:0:205976:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.534402:0:205976:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069c80. 00010000:00000010:2.1:1786173838.551388:0:205977:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e900. 00010000:00000010:2.1:1786173838.551403:0:205977:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.551408:0:205977:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069c80. 00010000:00000010:2.1:1786173838.566375:0:205978:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e900. 00010000:00000010:2.1:1786173838.566383:0:205978:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.566386:0:205978:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069c80. 00010000:00000010:2.1:1786173838.582390:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e900. 00010000:00000010:2.1:1786173838.582396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.582399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069c80. 00010000:00000010:2.1:1786173838.588413:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e300. 00010000:00000010:2.1:1786173838.588424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720ee00. 00010000:00000010:2.1:1786173838.588430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373a00. 00000080:00000010:3.1:1786173838.601376:0:205980:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069b20. 00010000:00000010:3.1:1786173838.609387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb800. 00010000:00000010:3.1:1786173838.609397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00000080:00000010:3.1:1786173838.609405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaa60. 00010000:00000010:3.1:1786173838.618885:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebc00. 00010000:00000010:3.1:1786173838.618910:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00000080:00000010:3.1:1786173838.618915:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaec0. 00010000:00000010:0.1:1786173838.628388:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8f00. 00010000:00000010:0.1:1786173838.628396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10561000. 00000080:00000010:0.1:1786173838.628400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e312884c0. 00010000:00000010:3.1:1786173838.645376:0:205984:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebc00. 00010000:00000010:3.1:1786173838.645380:0:205984:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00000080:00000010:3.1:1786173838.645383:0:205984:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca160. 00010000:00000010:3.1:1786173838.653387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb800. 00010000:00000010:3.1:1786173838.653398:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb100. 00010000:00000010:3.1:1786173838.653401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00000080:00000010:3.1:1786173838.659379:0:205985:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaa60. 00010000:00000010:3.1:1786173838.673374:0:205986:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb100. 00010000:00000010:3.1:1786173838.673379:0:205986:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00000080:00000010:3.1:1786173838.673384:0:205986:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaa60. 00010000:00000010:3.1:1786173838.682390:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb800. 00010000:00000010:3.1:1786173838.682396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c600. 00000080:00000010:3.1:1786173838.682399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaec0. 00010000:00000010:0.1:1786173838.699783:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8f00. 00010000:00000010:0.1:1786173838.699809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10561000. 00000080:00000010:0.1:1786173838.699814:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e312887e0. 00010000:00000010:3.1:1786173838.708669:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb800. 00010000:00000010:3.1:1786173838.708673:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c600. 00000080:00000010:3.1:1786173838.708676:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca160. 00010000:00000010:3.1:1786173838.716537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173838.720387:0:205990:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173838.720395:0:205990:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00000080:00000010:3.1:1786173838.724392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca400. 00010000:00000010:3.1:1786173838.736380:0:205991:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173838.736387:0:205991:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00000080:00000010:3.1:1786173838.736392:0:205991:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaec0. 00010000:00000010:3.1:1786173838.744384:0:205992:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173838.744387:0:205992:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173838.744390:0:205992:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca3a0. 00010000:00000010:3.1:1786173838.756380:0:205993:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173838.756384:0:205993:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173838.756389:0:205993:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaec0. 00010000:00000010:3.1:1786173838.764456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173838.764461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00000080:00000010:3.1:1786173838.764464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaa60. 00010000:00000010:3.1:1786173838.774496:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2900. 00010000:00000010:3.1:1786173838.774500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00010000:00000010:3.1:1786173838.778379:0:205995:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173838.778385:0:205995:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173838.778388:0:205995:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173838.782393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaec0. 00010000:00000010:3.1:1786173838.795404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173838.795412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173838.795417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca400. 00010000:00000010:2.1:1786173838.808412:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720ee00. 00010000:00000010:2.1:1786173838.808427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373a00. 00000080:00000010:2.1:1786173838.808435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069c80. 00010000:00000010:2.1:1786173838.821395:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e300. 00010000:00000010:2.1:1786173838.821403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.821406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069e20. 00010000:00000010:2.1:1786173838.834394:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720ee00. 00010000:00000010:2.1:1786173838.834402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373a00. 00000080:00000010:2.1:1786173838.834406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069aa0. 00010000:00000010:2.1:1786173838.842405:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2500. 00010000:00000010:2.1:1786173838.842410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44000. 00010000:00000010:2.1:1786173838.871415:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e300. 00010000:00000010:2.1:1786173838.871428:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e900. 00010000:00000010:2.1:1786173838.871438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.880380:0:206003:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069c80. 00010000:00000010:2.1:1786173838.892394:0:206004:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e200. 00010000:00000010:2.1:1786173838.892403:0:206004:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.892407:0:206004:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069c80. 00010000:00000010:2.1:1786173838.904393:0:206005:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e200. 00010000:00000010:2.1:1786173838.904401:0:206005:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.904406:0:206005:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069c80. 00010000:00000010:2.1:1786173838.922402:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e200. 00010000:00000010:2.1:1786173838.922410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.922416:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069c80. 00010000:00000010:3.1:1786173838.931380:0:206007:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173838.931388:0:206007:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173838.931392:0:206007:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca460. 00010000:00000010:3.1:1786173838.943382:0:206007:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173838.951392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173838.951402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173838.959393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca460. 00010000:00000010:2.1:1786173838.963400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e200. 00010000:00000010:2.1:1786173838.963411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173838.963418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069c80. 00010000:00000010:3.1:1786173838.975394:0:206010:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173838.975406:0:206010:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173838.975411:0:206010:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaec0. 00010000:00000010:3.1:1786173838.987423:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173838.987433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173838.987439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaec0. 00010000:00000010:0.1:1786173839.008393:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8f00. 00010000:00000010:0.1:1786173839.008399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10561000. 00000080:00000010:0.1:1786173839.008402:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288920. 00010000:00000010:0.1:1786173839.016696:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8d00. 00010000:00000010:0.1:1786173839.016707:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8400. 00010000:00000010:0.1:1786173839.016710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10560a00. 00000080:00000010:2.1:1786173839.024392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e312887e0. 00010000:00000010:2.1:1786173839.036401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e200. 00010000:00000010:2.1:1786173839.036411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173839.036417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069220. 00010000:00000010:2.1:1786173839.052389:0:206015:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230a00. 00010000:00000010:2.1:1786173839.052399:0:206015:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc5200. 00000080:00000010:2.1:1786173839.052409:0:206015:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09323120. 00010000:00000010:2.1:1786173839.060412:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e200. 00010000:00000010:2.1:1786173839.060419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173839.060423:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069aa0. 00010000:00000010:2.1:1786173839.071554:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70eb00. 00010000:00000010:2.1:1786173839.071565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373c00. 00000080:00000010:2.1:1786173839.071570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069c80. 00010000:00000010:2.1:1786173839.086391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e200. 00010000:00000010:2.1:1786173839.086397:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70eb00. 00010000:00000010:2.1:1786173839.086400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173839.098386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069aa0. 00010000:00000010:3.1:1786173839.107378:0:206019:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173839.107383:0:206019:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173839.107387:0:206019:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca460. 00010000:00000010:3.1:1786173839.115579:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173839.115584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00000080:00000010:3.1:1786173839.115587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaec0. 00010000:00000010:3.1:1786173839.123392:0:206021:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173839.123401:0:206021:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173839.123404:0:206021:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca300. 00010000:00000010:3.1:1786173839.144432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173839.144439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173839.144442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca300. 00010000:00000010:3.1:1786173839.150716:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173839.154388:0:206023:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebb00. 00000080:00000010:2.1:1786173839.154395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaec0. 00010000:00000010:3.1:1786173839.154397:0:206023:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00010000:00000010:3.1:1786173839.170426:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebb00. 00010000:00000010:3.1:1786173839.170433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00000080:00000010:3.1:1786173839.170436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca300. 00010000:00000010:3.1:1786173839.182392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173839.182397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173839.182401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca700. 00010000:00000010:3.1:1786173839.194391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebb00. 00010000:00000010:3.1:1786173839.194398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00000080:00000010:3.1:1786173839.194404:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca0a0. 00010000:00000010:2.1:1786173839.212412:0:206027:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230f00. 00010000:00000010:2.1:1786173839.212416:0:206027:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70eb00. 00010000:00000010:2.1:1786173839.212420:0:206027:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc5800. 00010000:00000010:3.1:1786173839.213383:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173839.213389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173839.213392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca300. 00000080:00000010:2.1:1786173839.233387:0:206029:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e09323f80. 00010000:00000010:2.1:1786173839.237672:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70eb00. 00010000:00000010:2.1:1786173839.237675:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372600. 00000080:00000010:2.1:1786173839.237678:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069220. 00010000:00000010:2.1:1786173839.245387:0:206030:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.245395:0:206030:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373e00. 00000080:00000010:2.1:1786173839.245399:0:206030:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069d20. 00010000:00000010:2.1:1786173839.258386:0:206031:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.258395:0:206031:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373e00. 00000080:00000010:2.1:1786173839.258400:0:206031:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069d20. 00010000:00000010:2.1:1786173839.271378:0:206032:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.271386:0:206032:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373e00. 00000080:00000010:2.1:1786173839.271390:0:206032:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ca0. 00010000:00000010:2.1:1786173839.287427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.291383:0:206033:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70eb00. 00010000:00000010:2.1:1786173839.291393:0:206033:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373e00. 00000080:00000010:2.1:1786173839.295386:0:206033:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ca0. 00010000:00000010:2.1:1786173839.307392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70eb00. 00010000:00000010:2.1:1786173839.307398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373e00. 00000080:00000010:2.1:1786173839.307402:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069f00. 00010000:00000010:2.1:1786173839.315418:0:206035:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.315423:0:206035:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00000080:00000010:2.1:1786173839.319392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069220. 00010000:00000010:2.1:1786173839.331407:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.331414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00000080:00000010:2.1:1786173839.331418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069980. 00010000:00000010:2.1:1786173839.343392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70eb00. 00010000:00000010:2.1:1786173839.343400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373e00. 00000080:00000010:2.1:1786173839.343404:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069220. 00010000:00000010:2.1:1786173839.360392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.360397:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70eb00. 00010000:00000010:2.1:1786173839.360401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00000080:00000010:2.1:1786173839.364392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069980. 00010000:00000010:2.1:1786173839.375393:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.375405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44000. 00000080:00000010:2.1:1786173839.375413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ca0. 00010000:00000010:2.1:1786173839.383389:0:206040:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70eb00. 00010000:00000010:2.1:1786173839.383400:0:206040:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00000080:00000010:2.1:1786173839.383406:0:206040:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069f00. 00010000:00000010:2.1:1786173839.398393:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70eb00. 00010000:00000010:2.1:1786173839.398400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00000080:00000010:2.1:1786173839.398403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069f00. 00010000:00000010:3.1:1786173839.416402:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebb00. 00010000:00000010:3.1:1786173839.416411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:3.1:1786173839.416418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca300. 00010000:00000010:3.1:1786173839.425387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173839.433397:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebb00. 00010000:00000010:3.1:1786173839.433404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00000080:00000010:2.1:1786173839.439396:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca460. 00010000:00000010:3.1:1786173839.448401:0:206044:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70eb00. 00010000:00000010:3.1:1786173839.448407:0:206044:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00000080:00000010:3.1:1786173839.448411:0:206044:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069d20. 00010000:00000010:3.1:1786173839.470393:0:206046:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebb00. 00010000:00000010:3.1:1786173839.470404:0:206046:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00000080:00000010:3.1:1786173839.470410:0:206046:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca0a0. 00010000:00000010:3.1:1786173839.474643:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173839.474652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00000080:00000010:3.1:1786173839.474657:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcacc0. 00010000:00000010:3.1:1786173839.491384:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebb00. 00010000:00000010:3.1:1786173839.491391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00000080:00000010:3.1:1786173839.491394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcafc0. 00010000:00000010:3.1:1786173839.500429:0:206048:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebe00. 00010000:00000010:3.1:1786173839.500444:0:206048:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173839.500449:0:206048:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00010000:00000010:3.1:1786173839.518390:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebe00. 00010000:00000010:3.1:1786173839.518402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00000080:00000010:3.1:1786173839.518412:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca700. 00010000:00000010:2.1:1786173839.536384:0:206051:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.536388:0:206051:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00000080:00000010:2.1:1786173839.536390:0:206051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ca0. 00010000:00000010:2.1:1786173839.548384:0:206052:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.548395:0:206052:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00000080:00000010:2.1:1786173839.548400:0:206052:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069540. 00010000:00000010:2.1:1786173839.560398:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.563637:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00010000:00000010:2.1:1786173839.563645:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb700. 00010000:00000010:2.1:1786173839.563648:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122ca00. 00000080:00000010:2.1:1786173839.568379:0:206052:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069f00. 00000080:00000010:2.1:1786173839.568394:0:206052:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d74a40. 00010000:00000010:2.1:1786173839.568401:0:206052:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.572385:0:206052:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44000. 00010000:00000010:2.1:1786173839.572393:0:206052:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb400. 00010000:00000010:2.1:1786173839.572397:0:206052:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d600. 00000080:00000010:2.1:1786173839.577385:0:206052:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ca0. 00000080:00000010:2.1:1786173839.577402:0:206052:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d75280. 00010000:00000010:2.1:1786173839.580394:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.580402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00010000:00000010:2.1:1786173839.580408:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb300. 00010000:00000010:2.1:1786173839.580411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d200. 00000080:00000010:2.1:1786173839.584394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069220. 00000080:00000010:2.1:1786173839.584407:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d76b40. 00010000:00000010:2.1:1786173839.588375:0:206052:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.592374:0:206052:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44000. 00010000:00000010:2.1:1786173839.592379:0:206052:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb200. 00010000:00000010:2.1:1786173839.592381:0:206052:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d400. 00000080:00000010:2.1:1786173839.592383:0:206052:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ca0. 00000080:00000010:2.1:1786173839.596408:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d739c0. 00010000:00000010:2.1:1786173839.596417:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.596421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372c00. 00010000:00000010:2.1:1786173839.596424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e500. 00010000:00000010:2.1:1786173839.596427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372a00. 00000080:00000010:2.1:1786173839.600388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069220. 00000080:00000010:2.1:1786173839.600400:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9dac0. 00010000:00000010:2.1:1786173839.604389:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.604394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44000. 00010000:00000010:2.1:1786173839.604398:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e200. 00010000:00000010:2.1:1786173839.604400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373800. 00000080:00000010:2.1:1786173839.608387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ca0. 00000080:00000010:2.1:1786173839.608400:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9a100. 00010000:00000010:2.1:1786173839.612387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.612393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372a00. 00010000:00000010:2.1:1786173839.612396:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e000. 00010000:00000010:2.1:1786173839.612398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373000. 00000080:00000010:2.1:1786173839.616390:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069f00. 00000080:00000010:2.1:1786173839.616410:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9f380. 00010000:00000010:2.1:1786173839.620392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.620397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373800. 00010000:00000010:2.1:1786173839.620400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebc00. 00010000:00000010:2.1:1786173839.620402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c400. 00000080:00000010:2.1:1786173839.624376:0:206052:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069220. 00000080:00000010:2.1:1786173839.624386:0:206052:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15d72100. 00010000:00000010:2.1:1786173839.628387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.628391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373000. 00010000:00000010:2.1:1786173839.628394:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb800. 00010000:00000010:2.1:1786173839.628396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c600. 00000080:00000010:2.1:1786173839.632388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ca0. 00000080:00000010:2.1:1786173839.632399:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c2940. 00010000:00000010:2.1:1786173839.636396:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.636403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373800. 00010000:00000010:2.1:1786173839.636405:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb100. 00010000:00000010:2.1:1786173839.636408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c200. 00000080:00000010:2.1:1786173839.640383:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069f00. 00000080:00000010:2.1:1786173839.640393:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c1080. 00010000:00000010:2.1:1786173839.644411:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.644424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373000. 00010000:00000010:2.1:1786173839.644427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ee00. 00010000:00000010:2.1:1786173839.644429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372000. 00000080:00000010:2.1:1786173839.652395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069220. 00000080:00000010:2.1:1786173839.652410:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9a940. 00010000:00000010:2.1:1786173839.652418:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.660395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373800. 00010000:00000010:2.1:1786173839.660401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb500. 00010000:00000010:2.1:1786173839.660404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c800. 00000080:00000010:2.1:1786173839.664411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ca0. 00000080:00000010:2.1:1786173839.664434:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c5280. 00010000:00000010:2.1:1786173839.664440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ee00. 00010000:00000010:2.1:1786173839.664445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372000. 00010000:00000010:2.1:1786173839.664462:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8f00. 00010000:00000010:2.1:1786173839.664466:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10561000. 00000080:00000010:2.1:1786173839.668404:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069f00. 00000080:00000010:2.1:1786173839.668419:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e3a288000. 00010000:00000010:2.1:1786173839.668428:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e900. 00010000:00000010:2.1:1786173839.668433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373000. 00010000:00000010:2.1:1786173839.668436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e800. 00010000:00000010:2.1:1786173839.668439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373c00. 00000080:00000010:2.1:1786173839.672385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069220. 00000080:00000010:2.1:1786173839.672396:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e99080. 00010000:00000010:2.1:1786173839.676392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.676399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373800. 00010000:00000010:2.1:1786173839.676402:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb000. 00010000:00000010:2.1:1786173839.676404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122c000. 00000080:00000010:2.1:1786173839.680375:0:206052:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069540. 00000080:00000010:2.1:1786173839.680385:0:206052:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c4a40. 00010000:00000010:2.1:1786173839.680390:0:206052:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e800. 00010000:00000010:2.1:1786173839.684376:0:206052:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373c00. 00010000:00000010:2.1:1786173839.684381:0:206052:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e200. 00010000:00000010:2.1:1786173839.684383:0:206052:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173839.684385:0:206052:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069f00. 00000080:00000010:2.1:1786173839.688412:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9d280. 00010000:00000010:2.1:1786173839.688423:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.688427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373800. 00010000:00000010:2.1:1786173839.688429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e500. 00010000:00000010:2.1:1786173839.688432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372600. 00000080:00000010:2.1:1786173839.692389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069220. 00000080:00000010:2.1:1786173839.692401:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9ca40. 00010000:00000010:2.1:1786173839.700396:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e200. 00010000:00000010:2.1:1786173839.700402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00010000:00000010:2.1:1786173839.700404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ea00. 00010000:00000010:2.1:1786173839.700406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373e00. 00000080:00000010:2.1:1786173839.704392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069f00. 00000080:00000010:2.1:1786173839.704404:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e15e9c200. 00010000:00000010:2.1:1786173839.709401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e500. 00010000:00000010:2.1:1786173839.709410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372600. 00010000:00000010:2.1:1786173839.709414:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebd00. 00010000:00000010:2.1:1786173839.709418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122da00. 00000080:00000010:2.1:1786173839.709424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069540. 00000080:00000010:2.1:1786173839.709433:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c4200. 00010000:00000010:2.1:1786173839.709438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173839.714379:0:206055:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373800. 00010000:00000010:2.1:1786173839.714385:0:206055:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9ebb00. 00010000:00000010:2.1:1786173839.714388:0:206055:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122d000. 00000080:00000010:2.1:1786173839.714393:0:206055:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069220. 00000080:00000010:2.1:1786173839.718407:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9b9e020c6b40. 00010000:00000010:2.1:1786173839.997385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc4600. 00010000:00000010:3.1:1786173840.001395:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230600. 00010000:00000010:3.1:1786173840.001405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc4e00. 00010000:00000010:3.1:1786173842.341418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc4e00. 00010000:00000010:3.1:1786173842.345436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d230100. 00010000:00000010:3.1:1786173842.345444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc4200. 00010000:00000010:3.1:1786173845.022401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eba00. 00010000:00000010:3.1:1786173845.026392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d9eb600. 00010000:00000010:3.1:1786173845.026397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3122dc00. 00000080:00000010:3.1:1786173845.032405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca6a0. 00010000:00000010:2.1:1786173845.046410:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70ef00. 00010000:00000010:2.1:1786173845.046442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373c00. 00010000:00000010:2.1:1786173845.050380:0:206105:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372400. 00000080:00000010:2.1:1786173845.050387:0:206105:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069880. 00010000:00000010:2.1:1786173845.342399:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0a70e500. 00010000:00000010:3.1:1786173845.342405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc5200. 00010000:00000010:2.1:1786173845.342409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372a00. 00000800:00000200:3.1:1786173889.164528:0:2384:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:3.1:1786173889.164533:0:2384:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:3.1:1786173889.164536:0:2384:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1 00000800:00000200:3.1:1786173893.837282:0:2396:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173893.837324:0:2395:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173895.892885:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173895.892889:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 1, 1 secs later) 00000800:00000200:2.1:1786173895.893286:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173895.893288:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 1, 1 secs later) 00000800:00000200:2.1:1786173909.197474:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173909.197805:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173909.198033:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173909.198502:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173909.198679:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173914.317297:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173914.317304:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 8, 5 secs later) 00000800:00000200:2.1:1786173914.318174:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173914.318181:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 8, 5 secs later) 00000800:00000200:2.1:1786173914.319148:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173914.319150:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 8, 5 secs later) 00000800:00000200:2.1:1786173914.319339:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173914.319341:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 8, 5 secs later) 00000800:00000200:2.1:1786173914.319584:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173914.319587:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 8, 5 secs later) 00000800:00000200:2.1:1786173919.437765:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173919.438075:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173919.438200:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173919.438317:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173919.438480:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173924.557222:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173924.557231:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 16, 12 secs later) 00000800:00000200:2.1:1786173924.558099:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173924.558102:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 16, 12 secs later) 00000800:00000200:2.1:1786173924.558441:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173924.558444:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 16, 12 secs later) 00000800:00000200:2.1:1786173924.558717:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173924.558721:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 16, 12 secs later) 00000800:00000200:2.1:1786173924.558946:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173924.558948:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 16, 12 secs later) 00000800:00000200:2.1:1786173934.797246:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173934.797253:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 16, 2 secs later) 00000800:00000200:2.1:1786173934.797948:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173934.797951:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 16, 2 secs later) 00000800:00000200:2.1:1786173934.798265:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173934.798267:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 16, 2 secs later) 00000800:00000200:2.1:1786173934.798577:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173934.798579:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 16, 2 secs later) 00000800:00000200:2.1:1786173934.798804:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173934.798805:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 16, 2 secs later) 00000800:00000200:2.1:1786173950.157228:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173950.157236:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 32, 24 secs later) 00000800:00000200:2.1:1786173950.158071:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173950.158073:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 32, 24 secs later) 00000800:00000200:2.1:1786173950.158303:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173950.158306:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 32, 24 secs later) 00000800:00000200:2.1:1786173950.158594:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173950.158597:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 32, 24 secs later) 00000800:00000200:2.1:1786173950.158955:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173950.158957:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 32, 24 secs later) 00000800:00000200:2.1:1786173980.877450:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173980.877464:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786173980.878518:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173980.878521:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786173980.878952:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173980.878955:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786173980.879192:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173980.879194:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786173980.879557:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173980.879559:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786173991.117140:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173991.117147:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786173991.117915:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173991.117919:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786173991.118241:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173991.118244:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786173991.118561:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173991.118563:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786173991.118912:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786173991.118914:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174001.357489:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174001.357497:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174001.358531:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174001.358537:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174001.358873:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174001.358875:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174001.359056:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174001.359058:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174001.359411:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174001.359413:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174006.477020:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174006.477027:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174006.477681:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174006.477686:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174006.477900:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174006.477902:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174006.478084:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174006.478086:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174006.478351:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174006.478352:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174011.597138:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174011.597145:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174011.597742:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174011.597744:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174011.597928:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174011.597930:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174011.598180:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174011.598182:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174011.598439:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174011.598441:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174016.717179:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174016.717184:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174016.717990:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174016.717992:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174016.718272:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174016.718275:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174016.718577:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174016.718579:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174016.718853:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174016.718855:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174021.837075:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174021.837079:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174021.837734:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174021.837741:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174021.838095:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174021.838097:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174021.838408:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174021.838410:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174021.838596:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174021.838597:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174026.957039:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174026.957045:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174026.957711:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174026.957713:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174026.957939:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174026.957941:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174026.958151:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174026.958153:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174026.958333:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174026.958335:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174032.077196:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174032.077202:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174032.077740:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174032.077742:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174032.077991:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174032.077993:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174032.078180:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174032.078181:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174032.078379:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174032.078381:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174037.197169:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174037.197178:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786174037.197830:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174037.197834:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786174037.198171:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174037.198174:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786174037.198515:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174037.198518:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786174037.198818:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174037.198820:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786174042.317319:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174042.317992:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174042.318258:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174042.318484:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174042.318669:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174047.437525:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174047.437532:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786174047.438523:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174047.438526:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786174047.438818:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174047.438819:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786174047.439068:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174047.439070:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786174047.439253:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174047.439255:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786174052.557460:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174052.557468:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174052.558152:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174052.558154:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174052.558355:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174052.558373:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174052.558618:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174052.558620:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174052.558881:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174052.558882:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174057.677429:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174057.677442:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174057.678519:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174057.678522:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174057.678893:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174057.678897:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174057.679201:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174057.679203:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174057.679677:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174057.679695:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174062.797014:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174062.797020:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174062.797787:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174062.797789:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174062.798115:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174062.798118:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174062.798445:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174062.798448:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174062.798787:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174062.798790:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174088.397055:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174088.397060:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174088.397848:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174088.397851:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174088.398213:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174088.398214:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174088.398507:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174088.398509:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174088.398695:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174088.398697:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174124.237273:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174124.237282:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174124.237824:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174124.237826:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174124.238014:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174124.238015:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174124.238222:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174124.238224:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174124.238418:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174124.238420:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174129.357105:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174129.357110:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174129.357771:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174129.357773:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174129.358084:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174129.358086:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174129.358300:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174129.358302:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174129.358573:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174129.358575:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1786174160.077184:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174160.077189:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1786174160.077850:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174160.077852:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1786174160.078048:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174160.078049:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1786174160.078290:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174160.078292:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1786174160.078562:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174160.078564:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1786174165.197743:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174165.197775:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1786174165.199265:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174165.199271:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1786174165.199936:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174165.199940:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1786174165.200335:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174165.200340:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1786174165.200970:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174165.200974:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1786174170.317272:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174170.317283:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1786174170.318064:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174170.318067:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1786174170.318441:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174170.318444:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1786174170.318765:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174170.318768:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1786174170.319040:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174170.319044:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1786174175.438299:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174175.438530:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174175.438682:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174175.438797:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174175.438982:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174180.557105:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174180.557113:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1786174180.558427:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174180.558430:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1786174180.558711:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174180.558714:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1786174180.558971:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174180.558972:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1786174180.559154:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174180.559156:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1786174185.677567:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174185.677577:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1786174185.678711:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174185.678717:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1786174185.679215:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174185.679218:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1786174185.679643:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174185.679646:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1786174185.679943:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174185.679946:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1786174190.797468:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174190.797476:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1786174190.798685:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174190.798690:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1786174190.799118:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174190.799122:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1786174190.799593:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174190.799597:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1786174190.800065:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174190.800069:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1786174195.917101:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174195.917109:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1786174195.917923:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174195.917925:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1786174195.918199:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174195.918201:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1786174195.918478:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174195.918481:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1786174195.918744:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174195.918746:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1786174201.037031:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174201.037037:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1786174201.037713:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174201.037715:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1786174201.037931:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174201.037933:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1786174201.038130:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174201.038134:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1786174201.038466:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174201.038468:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1786174206.157089:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174206.157093:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1786174206.157714:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174206.157718:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1786174206.158189:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174206.158192:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1786174206.158508:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174206.158510:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1786174206.158691:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174206.158693:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1786174211.277292:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174211.277300:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1786174211.277995:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174211.277997:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1786174211.278218:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174211.278220:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1786174211.278480:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174211.278482:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1786174211.278663:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174211.278665:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1786174216.397040:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174216.397046:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1786174216.397771:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174216.397773:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1786174216.397966:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174216.397968:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1786174216.398166:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174216.398168:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1786174216.398491:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174216.398493:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1786174231.757700:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174231.757711:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1786174231.759042:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174231.759046:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1786174231.759842:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174231.759847:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1786174231.760422:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174231.760426:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1786174231.761144:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174231.761147:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1786174236.877272:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174236.877280:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1786174236.877881:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174236.877883:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1786174236.878096:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174236.878098:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1786174236.878280:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174236.878281:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1786174236.878496:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174236.878498:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1786174241.997262:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174241.997720:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174241.998037:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174241.998338:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:3.1:1786174241.998777:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174247.120142:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174247.120200:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174247.120781:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174247.120785:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174247.121167:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174247.121170:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174252.237067:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174252.237071:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174252.237764:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174252.237765:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174252.237985:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174252.237987:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174252.238228:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174252.238230:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174252.238510:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174252.238512:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174257.357907:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174257.357930:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174257.359172:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174257.359177:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174257.359748:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174257.359776:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174257.360276:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174257.360280:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174257.360820:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174257.360826:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174282.957190:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174282.957195:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174282.958031:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174282.958037:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174282.958478:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174282.958480:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174282.958982:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174282.958985:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174282.959186:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174282.959188:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174288.077250:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174288.077260:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174288.078197:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174288.078198:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174288.078415:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174288.078417:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174288.078685:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174288.078688:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174288.079001:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174288.079004:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174293.197080:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174293.197085:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174293.197767:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174293.197769:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174293.197966:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174293.197968:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174293.198157:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174293.198159:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174293.198385:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174293.198387:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174298.317104:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174298.317112:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174298.317652:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174298.317653:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174298.317845:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174298.317847:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174298.318057:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174298.318059:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174298.318248:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174298.318251:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174313.677999:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174313.678003:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174313.678387:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174313.678390:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174313.678738:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174313.678741:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174313.678976:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174313.678978:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174313.679164:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174313.679166:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174318.797294:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174318.797301:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174318.798041:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174318.798043:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174318.798358:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174318.798360:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174318.798703:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174318.798715:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174318.798937:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174318.798938:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174323.916979:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174323.916985:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174323.917677:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174323.917678:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174323.917865:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174323.917866:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174323.918044:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174323.918045:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174323.918314:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174323.918316:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174329.037426:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174329.037429:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174329.038632:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174329.038639:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174329.039152:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174329.039153:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174329.039424:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174329.039426:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174329.039616:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174329.039618:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174334.157613:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174334.157624:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174334.158854:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174334.158857:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174334.159250:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174334.159254:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174334.159639:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174334.159643:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174334.160141:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174334.160144:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174339.277603:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174339.277615:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786174339.278885:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174339.278888:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786174339.279294:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174339.279297:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786174339.279687:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174339.279691:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786174339.280259:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174339.280263:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786174344.397626:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174344.397639:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786174344.398843:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174344.398846:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786174344.399234:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174344.399238:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786174344.399625:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174344.399628:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786174344.400147:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174344.400150:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786174349.517471:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174349.517484:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174349.518648:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174349.518651:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174349.518987:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174349.518989:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174349.519278:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174349.519280:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174349.519846:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174349.519850:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174354.637015:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174354.637020:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174354.637593:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174354.637595:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174354.637782:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174354.637783:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174354.637980:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174354.637982:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174354.638220:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174354.638222:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174359.757661:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174359.757674:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174359.758701:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174359.758704:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174359.759119:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174359.759122:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174359.759576:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174359.759580:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174359.759968:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174359.759971:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174364.877292:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174364.877303:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174364.878260:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174364.878263:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174364.878512:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174364.878515:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174364.878759:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174364.878761:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174364.878950:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174364.878956:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174390.477809:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174390.477817:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174390.479491:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174390.479493:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174390.479817:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174390.479820:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174390.480086:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174390.480088:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174390.480572:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174390.480575:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174395.597201:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174395.597207:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174395.598043:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174395.598046:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174395.598262:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174395.598264:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174395.598492:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174395.598494:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174395.598783:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174395.598796:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174426.317444:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174426.317450:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174426.318731:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174426.318736:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174426.319260:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174426.319262:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174426.319558:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174426.319560:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174426.319769:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174426.319771:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174431.437082:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174431.437085:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786174431.437612:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174431.437614:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786174431.437837:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174431.437839:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786174431.438023:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174431.438024:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786174431.438229:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174431.438231:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000080:00000010:0.1:1786174432.340405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18de20. 00000080:00000010:0.1:1786174432.340420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d380. 00000080:00000010:0.1:1786174432.340427:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18df20. 00000080:00000010:0.1:1786174432.340430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca1e0. 00000080:00000010:0.1:1786174432.340433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca4a0. 00000080:00000010:0.1:1786174432.340435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca0c0. 00000080:00000010:0.1:1786174432.340438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaa00. 00000080:00000010:0.1:1786174432.340440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca440. 00000080:00000010:0.1:1786174432.340442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcad00. 00000080:00000010:0.1:1786174432.340445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca7c0. 00000080:00000010:0.1:1786174432.344389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca320. 00000080:00000010:0.1:1786174432.344396:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcae20. 00000080:00000010:0.1:1786174432.344399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca180. 00000080:00000010:0.1:1786174432.344401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaea0. 00000080:00000010:0.1:1786174432.344404:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca600. 00000080:00000010:0.1:1786174432.344406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca640. 00000080:00000010:0.1:1786174432.344409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d500. 00000080:00000010:0.1:1786174432.344412:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d720. 00000080:00000010:0.1:1786174432.344414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dda0. 00000080:00000010:0.1:1786174432.344419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d440. 00000080:00000010:0.1:1786174432.344421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d1a0. 00000080:00000010:0.1:1786174432.344424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d8e0. 00000080:00000010:0.1:1786174432.344427:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18db60. 00000080:00000010:0.1:1786174432.344429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dcc0. 00000080:00000010:0.1:1786174432.344431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d000. 00000080:00000010:0.1:1786174432.344433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d580. 00000080:00000010:0.1:1786174432.344435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18da80. 00000080:00000010:0.1:1786174432.344437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dee0. 00000080:00000010:0.1:1786174432.344440:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dc00. 00000080:00000010:0.1:1786174432.344441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d040. 00000080:00000010:0.1:1786174432.344443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d080. 00000080:00000010:0.1:1786174432.344445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dc60. 00000080:00000010:0.1:1786174432.344462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca9e0. 00000080:00000010:0.1:1786174432.344464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca720. 00000080:00000010:0.1:1786174432.344467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca800. 00000080:00000010:0.1:1786174432.344469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca580. 00000080:00000010:0.1:1786174432.344472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca380. 00000080:00000010:0.1:1786174432.344474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca540. 00000080:00000010:0.1:1786174432.344476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcaa40. 00000080:00000010:0.1:1786174432.344479:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca120. 00000080:00000010:0.1:1786174432.344481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e312886e0. 00000080:00000010:0.1:1786174432.344483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288700. 00010000:00000010:0.1:1786174432.344486:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03400. 00010000:00000010:0.1:1786174432.344489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c1a00. 00010000:00000010:0.1:1786174432.344492:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03800. 00010000:00000010:0.1:1786174432.344494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0400. 00010000:00000010:0.1:1786174432.344496:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03a00. 00010000:00000010:0.1:1786174432.344498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c1e00. 00010000:00000010:0.1:1786174432.344500:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2a00. 00010000:00000010:0.1:1786174432.344502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030400. 00010000:00000010:0.1:1786174432.344504:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2100. 00010000:00000010:0.1:1786174432.344506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030200. 00010000:00000010:0.1:1786174432.344508:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2200. 00010000:00000010:0.1:1786174432.344509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031a00. 00010000:00000010:0.1:1786174432.344511:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac100. 00010000:00000010:0.1:1786174432.344513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807e00. 00010000:00000010:0.1:1786174432.344515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac800. 00010000:00000010:0.1:1786174432.344517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807c00. 00010000:00000010:0.1:1786174432.344518:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac900. 00010000:00000010:0.1:1786174432.344520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807400. 00010000:00000010:0.1:1786174432.344522:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac200. 00010000:00000010:0.1:1786174432.344523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807000. 00010000:00000010:0.1:1786174432.344525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac600. 00010000:00000010:0.1:1786174432.344527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807a00. 00010000:00000010:0.1:1786174432.344529:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5500. 00010000:00000010:0.1:1786174432.344534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943e00. 00010000:00000010:0.1:1786174432.344537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11daca00. 00010000:00000010:0.1:1786174432.344538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943c00. 00010000:00000010:0.1:1786174432.344540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5000. 00010000:00000010:0.1:1786174432.344542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942e00. 00010000:00000010:0.1:1786174432.344544:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5e00. 00010000:00000010:0.1:1786174432.344545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2a00. 00010000:00000010:0.1:1786174432.344548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5c00. 00010000:00000010:0.1:1786174432.344549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45800. 00010000:00000010:0.1:1786174432.344561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b900. 00010000:00000010:0.1:1786174432.344563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44200. 00010000:00000010:0.1:1786174432.344565:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7ec00. 00010000:00000010:0.1:1786174432.344567:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45c00. 00010000:00000010:0.1:1786174432.344568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7eb00. 00010000:00000010:0.1:1786174432.344570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45600. 00010000:00000010:0.1:1786174432.344572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7ea00. 00010000:00000010:0.1:1786174432.344574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9800. 00010000:00000010:0.1:1786174432.344576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e700. 00010000:00000010:0.1:1786174432.344578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9600. 00010000:00000010:0.1:1786174432.344579:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e800. 00010000:00000010:0.1:1786174432.344581:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9400. 00010000:00000010:0.1:1786174432.344582:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e300. 00010000:00000010:0.1:1786174432.344584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb8200. 00010000:00000010:0.1:1786174432.344585:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e100. 00010000:00000010:0.1:1786174432.344587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9000. 00010000:00000010:0.1:1786174432.344588:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03b00. 00010000:00000010:0.1:1786174432.344590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171dc00. 00010000:00000010:0.1:1786174432.344592:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e138abd00. 00010000:00000010:0.1:1786174432.344594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171d600. 00010000:00000010:0.1:1786174432.344596:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328afe00. 00010000:00000010:0.1:1786174432.344598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171ce00. 00010000:00000010:0.1:1786174432.344600:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e138ab200. 00010000:00000010:0.1:1786174432.344605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171c400. 00010000:00000010:0.1:1786174432.344607:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b0f100. 00010000:00000010:0.1:1786174432.344609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fada000. 00010000:00000010:0.1:1786174432.344612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af500. 00010000:00000010:0.1:1786174432.344614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadbc00. 00010000:00000010:0.1:1786174432.344615:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b0fe00. 00010000:00000010:0.1:1786174432.344617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadbe00. 00010000:00000010:0.1:1786174432.344618:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328aff00. 00010000:00000010:0.1:1786174432.344620:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fada200. 00010000:00000010:0.1:1786174432.344622:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af600. 00010000:00000010:0.1:1786174432.344623:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadb600. 00010000:00000010:0.1:1786174432.344625:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328afd00. 00010000:00000010:0.1:1786174432.344626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ace00. 00010000:00000010:0.1:1786174432.344628:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af400. 00010000:00000010:0.1:1786174432.344630:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ac200. 00010000:00000010:0.1:1786174432.344631:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af100. 00010000:00000010:0.1:1786174432.344633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ad600. 00010000:00000010:0.1:1786174432.344635:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072ddc00. 00010000:00000010:0.1:1786174432.344652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ac400. 00010000:00000010:0.1:1786174432.344654:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af200. 00010000:00000010:0.1:1786174432.344655:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ada00. 00010000:00000010:0.1:1786174432.344657:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dda00. 00010000:00000010:0.1:1786174432.344659:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ac000. 00010000:00000010:0.1:1786174432.344670:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072ddb00. 00010000:00000010:0.1:1786174432.344672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd2400. 00010000:00000010:0.1:1786174432.344674:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd400. 00010000:00000010:0.1:1786174432.344676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd2000. 00010000:00000010:0.1:1786174432.344678:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459800. 00010000:00000010:0.1:1786174432.344680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3a00. 00010000:00000010:0.1:1786174432.344681:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd900. 00010000:00000010:0.1:1786174432.344683:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3600. 00010000:00000010:0.1:1786174432.344685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459500. 00010000:00000010:0.1:1786174432.344689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3200. 00010000:00000010:0.1:1786174432.344691:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459000. 00010000:00000010:0.1:1786174432.344693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975000. 00010000:00000010:0.1:1786174432.344695:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459700. 00010000:00000010:0.1:1786174432.344697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31994a00. 00010000:00000010:0.1:1786174432.344699:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459100. 00010000:00000010:0.1:1786174432.344700:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31994600. 00010000:00000010:0.1:1786174432.344702:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459f00. 00010000:00000010:0.1:1786174432.344704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31994c00. 00010000:00000010:0.1:1786174432.344705:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459e00. 00010000:00000010:0.1:1786174432.344707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31994e00. 00010000:00000010:0.1:1786174432.344708:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418200. 00010000:00000010:0.1:1786174432.344710:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31994200. 00010000:00000010:0.1:1786174432.344712:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418600. 00010000:00000010:0.1:1786174432.344713:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4800. 00010000:00000010:0.1:1786174432.344715:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418000. 00010000:00000010:0.1:1786174432.344717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974e00. 00010000:00000010:0.1:1786174432.344718:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418d00. 00010000:00000010:0.1:1786174432.344720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975600. 00010000:00000010:0.1:1786174432.344721:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418900. 00010000:00000010:0.1:1786174432.344723:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975400. 00010000:00000010:0.1:1786174432.344725:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418e00. 00010000:00000010:0.1:1786174432.344727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974c00. 00010000:00000010:0.1:1786174432.344729:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9200. 00010000:00000010:0.1:1786174432.344731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974600. 00010000:00000010:0.1:1786174432.344733:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9b00. 00010000:00000010:0.1:1786174432.344734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975a00. 00010000:00000010:0.1:1786174432.344736:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9300. 00010000:00000010:0.1:1786174432.344737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974400. 00010000:00000010:0.1:1786174432.344739:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9800. 00010000:00000010:0.1:1786174432.344741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974a00. 00010000:00000010:0.1:1786174432.344742:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9000. 00010000:00000010:0.1:1786174432.344747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974200. 00010000:00000010:0.1:1786174432.344748:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9c00. 00010000:00000010:0.1:1786174432.344750:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5600. 00010000:00000010:0.1:1786174432.344752:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68ba00. 00010000:00000010:0.1:1786174432.344754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2400. 00010000:00000010:0.1:1786174432.344756:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b600. 00010000:00000010:0.1:1786174432.344758:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc3400. 00010000:00000010:0.1:1786174432.344759:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68bd00. 00010000:00000010:0.1:1786174432.344761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2c00. 00010000:00000010:0.1:1786174432.344762:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68bb00. 00010000:00000010:0.1:1786174432.344764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc3c00. 00010000:00000010:0.1:1786174432.344765:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68bf00. 00010000:00000010:0.1:1786174432.344767:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2000. 00010000:00000010:0.1:1786174432.344769:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa500. 00010000:00000010:0.1:1786174432.344770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343200. 00010000:00000010:0.1:1786174432.344772:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa300. 00010000:00000010:0.1:1786174432.344774:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342c00. 00010000:00000010:0.1:1786174432.344776:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aad00. 00010000:00000010:0.1:1786174432.344777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342200. 00010000:00000010:0.1:1786174432.344779:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa100. 00010000:00000010:0.1:1786174432.344780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343800. 00010000:00000010:0.1:1786174432.344782:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa600. 00010000:00000010:0.1:1786174432.344783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342a00. 00010000:00000010:0.1:1786174432.344785:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3900. 00010000:00000010:0.1:1786174432.344787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1200. 00000800:00000200:2.1:1786174436.557810:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174436.557817:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174436.559356:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174436.559359:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174436.559712:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174436.559714:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174436.559959:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174436.559961:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174436.560421:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174436.560423:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000080:00000010:0.1:1786174439.519422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069820. 00000080:00000010:0.1:1786174439.523430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069d00. 00000080:00000010:0.1:1786174439.523441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069180. 00000080:00000010:0.1:1786174439.523475:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069920. 00000080:00000010:0.1:1786174439.523480:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069520. 00000080:00000010:0.1:1786174439.523485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e110692e0. 00000080:00000010:0.1:1786174439.523504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069580. 00000080:00000010:0.1:1786174439.523512:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e11069ce0. 00000080:00000010:0.1:1786174439.523536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e110691a0. 00000080:00000010:0.1:1786174439.523565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d900. 00000080:00000010:0.1:1786174439.523583:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dd00. 00000080:00000010:0.1:1786174439.523604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d240. 00000080:00000010:0.1:1786174439.523608:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dce0. 00000080:00000010:0.1:1786174439.523884:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e093230a0. 00000080:00000010:0.1:1786174439.523891:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcae40. 00000080:00000010:0.1:1786174439.523895:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcace0. 00000080:00000010:0.1:1786174439.523917:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca5a0. 00000080:00000010:0.1:1786174439.523930:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcade0. 00000080:00000010:0.1:1786174439.523949:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca480. 00000080:00000010:0.1:1786174439.523952:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca280. 00000080:00000010:0.1:1786174439.524264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca660. 00000080:00000010:0.1:1786174439.524270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcabe0. 00000080:00000010:0.1:1786174439.524287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca1c0. 00000080:00000010:0.1:1786174439.524301:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca7e0. 00000080:00000010:0.1:1786174439.524304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca6c0. 00000080:00000010:0.1:1786174439.524317:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca4c0. 00000080:00000010:0.1:1786174439.524334:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca360. 00000080:00000010:0.1:1786174439.524347:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca760. 00000080:00000010:0.1:1786174439.524382:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca960. 00000080:00000010:0.1:1786174439.524386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca8e0. 00000080:00000010:0.1:1786174439.524401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca3e0. 00000080:00000010:0.1:1786174439.524415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcae60. 00000080:00000010:0.1:1786174439.524436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca080. 00000080:00000010:0.1:1786174439.524462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d020. 00000080:00000010:0.1:1786174439.524467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18de80. 00000080:00000010:0.1:1786174439.524515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d9c0. 00000080:00000010:0.1:1786174439.524533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d4c0. 00000080:00000010:0.1:1786174439.524546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d7e0. 00000080:00000010:0.1:1786174439.524563:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dde0. 00000080:00000010:0.1:1786174439.524566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d9a0. 00000080:00000010:0.1:1786174439.524584:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d920. 00000080:00000010:0.1:1786174439.524587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d6c0. 00000080:00000010:0.1:1786174439.524589:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d0e0. 00000080:00000010:0.1:1786174439.524604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18de00. 00000080:00000010:0.1:1786174439.524618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d600. 00000080:00000010:0.1:1786174439.524621:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d060. 00000080:00000010:0.1:1786174439.524625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18de60. 00000080:00000010:0.1:1786174439.524641:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dbe0. 00000080:00000010:0.1:1786174439.524644:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d860. 00000080:00000010:0.1:1786174439.524658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d120. 00000080:00000010:0.1:1786174439.524673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d200. 00000080:00000010:0.1:1786174439.524691:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18dae0. 00000080:00000010:0.1:1786174439.524703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18de40. 00000080:00000010:0.1:1786174439.524714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e1e18d680. 00000080:00000010:0.1:1786174439.524729:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca3c0. 00000080:00000010:0.1:1786174439.524785:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcab20. 00000080:00000010:0.1:1786174439.524790:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bca1a0. 00000080:00000010:0.1:1786174439.524809:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcab60. 00000080:00000010:0.1:1786174439.524813:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e07bcad40. 00000080:00000010:0.1:1786174439.524830:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288d40. 00000080:00000010:0.1:1786174439.524845:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288820. 00000080:00000010:0.1:1786174439.524848:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288ca0. 00000080:00000010:0.1:1786174439.524852:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288620. 00000080:00000010:0.1:1786174439.524867:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e312882e0. 00000080:00000010:0.1:1786174439.524884:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9b9e31288460. 00010000:00000010:0.1:1786174439.524888:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc5c00. 00010000:00000010:0.1:1786174439.524893:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720eb00. 00010000:00000010:0.1:1786174439.524897:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372e00. 00010000:00000010:0.1:1786174439.524900:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e700. 00010000:00000010:0.1:1786174439.524902:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373200. 00010000:00000010:0.1:1786174439.524905:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720ed00. 00010000:00000010:0.1:1786174439.524908:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373600. 00010000:00000010:0.1:1786174439.524910:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e800. 00010000:00000010:0.1:1786174439.524912:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e553800. 00010000:00000010:0.1:1786174439.524919:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171d000. 00010000:00000010:0.1:1786174439.524921:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e600. 00010000:00000010:0.1:1786174439.524923:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e553a00. 00010000:00000010:0.1:1786174439.524925:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720ea00. 00010000:00000010:0.1:1786174439.524929:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e552a00. 00010000:00000010:0.1:1786174439.524931:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f400. 00010000:00000010:0.1:1786174439.524934:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e553e00. 00010000:00000010:0.1:1786174439.524937:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818fb00. 00010000:00000010:0.1:1786174439.524939:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e552600. 00010000:00000010:0.1:1786174439.524941:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818fc00. 00010000:00000010:0.1:1786174439.524943:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e553c00. 00010000:00000010:0.1:1786174439.524947:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818ff00. 00010000:00000010:0.1:1786174439.524949:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e552800. 00010000:00000010:0.1:1786174439.524951:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818fe00. 00010000:00000010:0.1:1786174439.524953:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e553400. 00010000:00000010:0.1:1786174439.524955:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f000. 00010000:00000010:0.1:1786174439.524957:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e552200. 00010000:00000010:0.1:1786174439.524959:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f100. 00010000:00000010:0.1:1786174439.524962:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0e00. 00010000:00000010:0.1:1786174439.524965:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03c00. 00010000:00000010:0.1:1786174439.524968:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0a00. 00010000:00000010:0.1:1786174439.524970:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c1200. 00010000:00000010:0.1:1786174439.524973:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03300. 00010000:00000010:0.1:1786174439.524975:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0200. 00010000:00000010:0.1:1786174439.524977:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03700. 00010000:00000010:0.1:1786174439.524979:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c1800. 00010000:00000010:0.1:1786174439.524981:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03f00. 00010000:00000010:0.1:1786174439.524983:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c1000. 00010000:00000010:0.1:1786174439.524985:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0800. 00010000:00000010:0.1:1786174439.525004:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03600. 00010000:00000010:0.1:1786174439.525008:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0000. 00010000:00000010:0.1:1786174439.525011:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0600. 00010000:00000010:0.1:1786174439.525013:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03200. 00010000:00000010:0.1:1786174439.525015:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c1c00. 00010000:00000010:0.1:1786174439.525017:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03900. 00010000:00000010:0.1:1786174439.525019:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c0c00. 00010000:00000010:0.1:1786174439.525022:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03000. 00010000:00000010:0.1:1786174439.525025:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171c000. 00010000:00000010:0.1:1786174439.525027:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171c600. 00010000:00000010:0.1:1786174439.525030:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03e00. 00010000:00000010:0.1:1786174439.525032:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171de00. 00010000:00000010:0.1:1786174439.525035:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2f00. 00010000:00000010:0.1:1786174439.525037:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030000. 00010000:00000010:0.1:1786174439.525040:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2b00. 00010000:00000010:0.1:1786174439.525042:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031800. 00010000:00000010:0.1:1786174439.525044:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2c00. 00010000:00000010:0.1:1786174439.525046:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031e00. 00010000:00000010:0.1:1786174439.525048:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2800. 00010000:00000010:0.1:1786174439.525051:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031c00. 00010000:00000010:0.1:1786174439.525054:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2d00. 00010000:00000010:0.1:1786174439.525057:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030600. 00010000:00000010:0.1:1786174439.525061:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00030a00. 00010000:00000010:0.1:1786174439.525064:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2000. 00010000:00000010:0.1:1786174439.525068:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e00031200. 00010000:00000010:0.1:1786174439.525070:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e319b2600. 00010000:00000010:0.1:1786174439.525074:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807800. 00010000:00000010:0.1:1786174439.525078:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f806800. 00010000:00000010:0.1:1786174439.525081:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac400. 00010000:00000010:0.1:1786174439.525085:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f806200. 00010000:00000010:0.1:1786174439.525089:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f806a00. 00010000:00000010:0.1:1786174439.525092:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5700. 00010000:00000010:0.1:1786174439.525097:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f806400. 00010000:00000010:0.1:1786174439.525101:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac700. 00010000:00000010:0.1:1786174439.525105:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f806e00. 00010000:00000010:0.1:1786174439.525109:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac000. 00010000:00000010:0.1:1786174439.525112:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807200. 00010000:00000010:0.1:1786174439.525116:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f807600. 00010000:00000010:0.1:1786174439.525120:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dacb00. 00010000:00000010:0.1:1786174439.525123:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f806000. 00010000:00000010:0.1:1786174439.525125:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac300. 00010000:00000010:0.1:1786174439.525128:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f806600. 00010000:00000010:0.1:1786174439.525131:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dacc00. 00010000:00000010:0.1:1786174439.525133:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f806c00. 00010000:00000010:0.1:1786174439.525142:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dac500. 00010000:00000010:0.1:1786174439.525145:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943000. 00010000:00000010:0.1:1786174439.525147:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e11dace00. 00010000:00000010:0.1:1786174439.525149:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942a00. 00010000:00000010:0.1:1786174439.525152:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b000. 00010000:00000010:0.1:1786174439.525155:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943800. 00010000:00000010:0.1:1786174439.525157:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5b00. 00010000:00000010:0.1:1786174439.525160:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942400. 00010000:00000010:0.1:1786174439.525163:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943600. 00010000:00000010:0.1:1786174439.525165:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5900. 00010000:00000010:0.1:1786174439.525167:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942000. 00010000:00000010:0.1:1786174439.525169:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5600. 00010000:00000010:0.1:1786174439.525171:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942600. 00010000:00000010:0.1:1786174439.525173:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5100. 00010000:00000010:0.1:1786174439.525175:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942c00. 00010000:00000010:0.1:1786174439.525177:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5f00. 00010000:00000010:0.1:1786174439.525179:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31942800. 00010000:00000010:0.1:1786174439.525181:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5300. 00010000:00000010:0.1:1786174439.525185:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44600. 00010000:00000010:0.1:1786174439.525187:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5a00. 00010000:00000010:0.1:1786174439.525189:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31943a00. 00010000:00000010:0.1:1786174439.525192:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44400. 00010000:00000010:0.1:1786174439.525194:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5800. 00010000:00000010:0.1:1786174439.525197:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45000. 00010000:00000010:0.1:1786174439.525199:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44a00. 00010000:00000010:0.1:1786174439.525201:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5400. 00010000:00000010:0.1:1786174439.525203:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45a00. 00010000:00000010:0.1:1786174439.525205:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fab5d00. 00010000:00000010:0.1:1786174439.525208:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45200. 00010000:00000010:0.1:1786174439.525211:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e200. 00010000:00000010:0.1:1786174439.525214:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45400. 00010000:00000010:0.1:1786174439.525217:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf45e00. 00010000:00000010:0.1:1786174439.525219:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e600. 00010000:00000010:0.1:1786174439.525221:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44800. 00010000:00000010:0.1:1786174439.525223:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44c00. 00010000:00000010:0.1:1786174439.525225:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7ed00. 00010000:00000010:0.1:1786174439.525227:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bf44e00. 00010000:00000010:0.1:1786174439.525229:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e500. 00010000:00000010:0.1:1786174439.525233:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9a00. 00010000:00000010:0.1:1786174439.525235:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e900. 00010000:00000010:0.1:1786174439.525237:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb8e00. 00010000:00000010:0.1:1786174439.525239:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e400. 00010000:00000010:0.1:1786174439.525241:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb8400. 00010000:00000010:0.1:1786174439.525243:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7ee00. 00010000:00000010:0.1:1786174439.525245:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb8a00. 00010000:00000010:0.1:1786174439.525247:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7e000. 00010000:00000010:0.1:1786174439.525249:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9200. 00010000:00000010:0.1:1786174439.525252:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa900. 00010000:00000010:0.1:1786174439.525255:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb8000. 00010000:00000010:0.1:1786174439.525257:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0fb7ef00. 00010000:00000010:0.1:1786174439.525268:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9e00. 00010000:00000010:0.1:1786174439.525270:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb8600. 00010000:00000010:0.1:1786174439.525273:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b200. 00010000:00000010:0.1:1786174439.525275:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb9c00. 00010000:00000010:0.1:1786174439.525278:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb8800. 00010000:00000010:0.1:1786174439.525280:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68be00. 00010000:00000010:0.1:1786174439.525282:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0bfb8c00. 00010000:00000010:0.1:1786174439.525307:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc3000. 00010000:00000010:0.1:1786174439.525312:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b300. 00010000:00000010:0.1:1786174439.525315:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc3800. 00010000:00000010:0.1:1786174439.525317:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e138abb00. 00010000:00000010:0.1:1786174439.525320:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171c200. 00010000:00000010:0.1:1786174439.525322:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171c800. 00010000:00000010:0.1:1786174439.525324:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b0f000. 00010000:00000010:0.1:1786174439.525327:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171cc00. 00010000:00000010:0.1:1786174439.525329:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e138aba00. 00010000:00000010:0.1:1786174439.525331:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171d400. 00010000:00000010:0.1:1786174439.525333:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e138ab900. 00010000:00000010:0.1:1786174439.525336:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171d200. 00010000:00000010:0.1:1786174439.525338:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e138ab000. 00010000:00000010:0.1:1786174439.525340:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171ca00. 00010000:00000010:0.1:1786174439.525342:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e138ab500. 00010000:00000010:0.1:1786174439.525344:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e3171d800. 00010000:00000010:0.1:1786174439.525346:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b0f600. 00010000:00000010:0.1:1786174439.525348:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadaa00. 00010000:00000010:0.1:1786174439.525351:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b0f500. 00010000:00000010:0.1:1786174439.525353:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadb000. 00010000:00000010:0.1:1786174439.525380:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd100. 00010000:00000010:0.1:1786174439.525385:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fada600. 00010000:00000010:0.1:1786174439.525388:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af300. 00010000:00000010:0.1:1786174439.525395:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadb200. 00010000:00000010:0.1:1786174439.525397:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fada800. 00010000:00000010:0.1:1786174439.525399:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328afb00. 00010000:00000010:0.1:1786174439.525402:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadac00. 00010000:00000010:0.1:1786174439.525404:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadae00. 00010000:00000010:0.1:1786174439.525406:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af900. 00010000:00000010:0.1:1786174439.525408:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadb400. 00010000:00000010:0.1:1786174439.525410:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328afa00. 00010000:00000010:0.1:1786174439.525414:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fada400. 00010000:00000010:0.1:1786174439.525418:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadba00. 00010000:00000010:0.1:1786174439.525423:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af000. 00010000:00000010:0.1:1786174439.525428:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0fadb800. 00010000:00000010:0.1:1786174439.525440:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ad400. 00010000:00000010:0.1:1786174439.525444:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af700. 00010000:00000010:0.1:1786174439.525448:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3000. 00010000:00000010:0.1:1786174439.525452:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328afc00. 00010000:00000010:0.1:1786174439.525455:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ad200. 00010000:00000010:0.1:1786174439.525459:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e328af800. 00010000:00000010:0.1:1786174439.525469:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261aca00. 00010000:00000010:0.1:1786174439.525474:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418f00. 00010000:00000010:0.1:1786174439.525478:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ad800. 00010000:00000010:0.1:1786174439.525482:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd500. 00010000:00000010:0.1:1786174439.525485:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ad000. 00010000:00000010:0.1:1786174439.525488:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd800. 00010000:00000010:0.1:1786174439.525490:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ac600. 00010000:00000010:0.1:1786174439.525493:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072ddd00. 00010000:00000010:0.1:1786174439.525495:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261acc00. 00010000:00000010:0.1:1786174439.525497:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261adc00. 00010000:00000010:0.1:1786174439.525501:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072ddf00. 00010000:00000010:0.1:1786174439.525503:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ade00. 00010000:00000010:0.1:1786174439.525505:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd200. 00010000:00000010:0.1:1786174439.525508:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd2a00. 00010000:00000010:0.1:1786174439.525510:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dde00. 00010000:00000010:0.1:1786174439.525512:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e261ac800. 00010000:00000010:0.1:1786174439.525519:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd000. 00010000:00000010:0.1:1786174439.525522:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd2600. 00010000:00000010:0.1:1786174439.525524:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd600. 00010000:00000010:0.1:1786174439.525526:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3c00. 00010000:00000010:0.1:1786174439.525529:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd2e00. 00010000:00000010:0.1:1786174439.525531:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd700. 00010000:00000010:0.1:1786174439.525533:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3e00. 00010000:00000010:0.1:1786174439.525535:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd2800. 00010000:00000010:0.1:1786174439.525537:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e072dd300. 00010000:00000010:0.1:1786174439.525540:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd2c00. 00010000:00000010:0.1:1786174439.525543:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459c00. 00010000:00000010:0.1:1786174439.525546:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3400. 00010000:00000010:0.1:1786174439.525548:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459b00. 00010000:00000010:0.1:1786174439.525550:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd2200. 00010000:00000010:0.1:1786174439.525553:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459a00. 00010000:00000010:0.1:1786174439.525555:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31bd3800. 00010000:00000010:0.1:1786174439.525561:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459200. 00010000:00000010:0.1:1786174439.525563:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995000. 00010000:00000010:0.1:1786174439.525566:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459d00. 00010000:00000010:0.1:1786174439.525568:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31994000. 00010000:00000010:0.1:1786174439.525570:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459300. 00010000:00000010:0.1:1786174439.525572:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31994400. 00010000:00000010:0.1:1786174439.525574:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459900. 00010000:00000010:0.1:1786174439.525576:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995c00. 00010000:00000010:0.1:1786174439.525580:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459600. 00010000:00000010:0.1:1786174439.525582:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995e00. 00010000:00000010:0.1:1786174439.525584:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e459400. 00010000:00000010:0.1:1786174439.525586:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31994800. 00010000:00000010:0.1:1786174439.525588:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418a00. 00010000:00000010:0.1:1786174439.525590:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995200. 00010000:00000010:0.1:1786174439.525592:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995600. 00010000:00000010:0.1:1786174439.525595:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418100. 00010000:00000010:0.1:1786174439.525597:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995400. 00010000:00000010:0.1:1786174439.525599:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418400. 00010000:00000010:0.1:1786174439.525601:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995800. 00010000:00000010:0.1:1786174439.525604:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418b00. 00010000:00000010:0.1:1786174439.525606:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975c00. 00010000:00000010:0.1:1786174439.525608:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418800. 00010000:00000010:0.1:1786174439.525610:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e31995a00. 00010000:00000010:0.1:1786174439.525616:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418c00. 00010000:00000010:0.1:1786174439.525619:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975e00. 00010000:00000010:0.1:1786174439.525621:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418700. 00010000:00000010:0.1:1786174439.525623:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f974000. 00010000:00000010:0.1:1786174439.525626:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e39418300. 00010000:00000010:0.1:1786174439.525628:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e0f975200. 00010000:00000010:0.1:1786174439.525631:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9e00. 00010000:00000010:0.1:1786174439.525633:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4600. 00010000:00000010:0.1:1786174439.525635:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5000. 00010000:00000010:0.1:1786174439.525637:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9f00. 00010000:00000010:0.1:1786174439.525639:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5200. 00010000:00000010:0.1:1786174439.525641:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4e00. 00010000:00000010:0.1:1786174439.525643:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9400. 00010000:00000010:0.1:1786174439.525645:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5e00. 00010000:00000010:0.1:1786174439.525647:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4c00. 00010000:00000010:0.1:1786174439.525650:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9900. 00010000:00000010:0.1:1786174439.525652:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b5800. 00010000:00000010:0.1:1786174439.525654:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4000. 00010000:00000010:0.1:1786174439.525656:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e261a9500. 00010000:00000010:0.1:1786174439.525658:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e093b4a00. 00010000:00000010:0.1:1786174439.525660:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b500. 00010000:00000010:0.1:1786174439.525663:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2e00. 00010000:00000010:0.1:1786174439.525665:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc3a00. 00010000:00000010:0.1:1786174439.525667:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b700. 00010000:00000010:0.1:1786174439.525669:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc3200. 00010000:00000010:0.1:1786174439.525672:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2200. 00010000:00000010:0.1:1786174439.525674:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b800. 00010000:00000010:0.1:1786174439.525676:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc3600. 00010000:00000010:0.1:1786174439.525678:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b100. 00010000:00000010:0.1:1786174439.525680:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2800. 00010000:00000010:0.1:1786174439.525682:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68b400. 00010000:00000010:0.1:1786174439.525684:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc3e00. 00010000:00000010:0.1:1786174439.525686:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e68bc00. 00010000:00000010:0.1:1786174439.525692:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc2600. 00010000:00000010:0.1:1786174439.525698:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa400. 00010000:00000010:0.1:1786174439.525702:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342400. 00010000:00000010:0.1:1786174439.525705:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa000. 00010000:00000010:0.1:1786174439.525707:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342e00. 00010000:00000010:0.1:1786174439.525710:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaf00. 00010000:00000010:0.1:1786174439.525712:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343400. 00010000:00000010:0.1:1786174439.525714:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343e00. 00010000:00000010:0.1:1786174439.525717:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aae00. 00010000:00000010:0.1:1786174439.525719:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342600. 00010000:00000010:0.1:1786174439.525721:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa200. 00010000:00000010:0.1:1786174439.525723:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342800. 00010000:00000010:0.1:1786174439.525725:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aaa00. 00010000:00000010:0.1:1786174439.525727:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343c00. 00010000:00000010:0.1:1786174439.525729:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa700. 00010000:00000010:0.1:1786174439.525731:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343a00. 00010000:00000010:0.1:1786174439.525733:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aac00. 00010000:00000010:0.1:1786174439.525735:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15342000. 00010000:00000010:0.1:1786174439.525737:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1d0aa800. 00010000:00000010:0.1:1786174439.525739:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e15343600. 00010000:00000010:0.1:1786174439.525741:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02fa8100. 00010000:00000010:0.1:1786174439.525744:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10560800. 00010000:00000010:0.1:1786174439.525747:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3800. 00010000:00000010:0.1:1786174439.525750:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10561400. 00010000:00000010:0.1:1786174439.525752:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10560200. 00010000:00000010:0.1:1786174439.525754:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3a00. 00010000:00000010:0.1:1786174439.525756:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e10561600. 00010000:00000010:0.1:1786174439.525758:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3b00. 00010000:00000010:0.1:1786174439.525761:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc0e00. 00010000:00000010:0.1:1786174439.525763:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1c00. 00010000:00000010:0.1:1786174439.525765:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3f00. 00010000:00000010:0.1:1786174439.525767:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc0400. 00010000:00000010:0.1:1786174439.525770:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3500. 00010000:00000010:0.1:1786174439.525773:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc0800. 00010000:00000010:0.1:1786174439.525775:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3e00. 00010000:00000010:0.1:1786174439.525778:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1600. 00010000:00000010:0.1:1786174439.525780:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3400. 00010000:00000010:0.1:1786174439.525782:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc0600. 00010000:00000010:0.1:1786174439.525785:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3600. 00010000:00000010:0.1:1786174439.525787:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1400. 00010000:00000010:0.1:1786174439.525789:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc0200. 00010000:00000010:0.1:1786174439.525791:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3100. 00010000:00000010:0.1:1786174439.525793:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1000. 00010000:00000010:0.1:1786174439.525795:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc1e00. 00010000:00000010:0.1:1786174439.525797:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e1e6f3200. 00010000:00000010:0.1:1786174439.525799:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9d0cdc0000. 00010000:00000010:0.1:1786174441.555415:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720e100. 00010000:00000010:0.1:1786174441.555426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372800. 00010000:00000010:0.1:1786174441.555430:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720ec00. 00010000:00000010:0.1:1786174441.555433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01373400. 00010000:00000010:0.1:1786174441.555436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0720ef00. 00010000:00000010:0.1:1786174441.555438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e01372200. 00010000:00000010:0.1:1786174441.555441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e02b03d00. 00010000:00000010:0.1:1786174441.555444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e552400. 00010000:00000010:0.1:1786174441.555448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818fd00. 00010000:00000010:0.1:1786174441.559422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e552000. 00010000:00000010:0.1:1786174441.559434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818fa00. 00010000:00000010:0.1:1786174441.559450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e552c00. 00010000:00000010:0.1:1786174441.559454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f500. 00010000:00000010:0.1:1786174441.559459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e552e00. 00010000:00000010:0.1:1786174441.559463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f300. 00010000:00000010:0.1:1786174441.559468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e553200. 00010000:00000010:0.1:1786174441.559472:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f800. 00010000:00000010:0.1:1786174441.559476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1e553600. 00010000:00000010:0.1:1786174441.559480:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f700. 00010000:00000010:0.1:1786174441.559486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c1400. 00010000:00000010:0.1:1786174441.559490:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9b9e0818f900. 00010000:00000010:0.1:1786174441.559494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9b9e1d9c1600. 00000800:00000200:2.1:1786174441.677602:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174441.678343:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174441.678859:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174441.679247:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174441.679587:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174446.797155:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174446.797163:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174446.798242:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174446.798245:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174446.798557:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174446.798559:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174446.798750:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174446.798752:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174446.798987:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174446.798989:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174451.917057:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174451.917066:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174451.917803:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174451.917806:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174451.918070:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174451.918074:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174451.918384:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174451.918386:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174451.918609:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174451.918611:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174457.037615:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174457.037624:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174457.038854:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174457.038863:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174457.039628:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174457.039632:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174457.040111:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174457.040114:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174457.040477:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174457.040480:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174462.157622:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174462.157633:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174462.158754:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174462.158757:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174462.159220:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174462.159224:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174462.159821:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174462.159826:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174462.160202:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174462.160205:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174467.277545:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174467.277555:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174467.278457:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174467.278460:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174467.278865:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174467.278867:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174467.279229:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174467.279232:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786174467.279586:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174467.279589:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174472.399024:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786174472.399271:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174472.399273:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786174477.517125:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174477.517132:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174477.517726:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174477.517727:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174477.517914:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174477.517916:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174477.518093:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174477.518094:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174477.518274:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174477.518276:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174482.637701:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174482.637717:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174482.638823:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174482.638827:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174482.639187:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174482.639190:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174482.639610:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174482.639613:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174482.640024:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174482.640027:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174487.757452:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174487.757468:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174487.758836:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174487.758837:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174487.759118:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174487.759120:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174487.759379:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174487.759382:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174487.759760:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174487.759772:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174492.877169:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174492.877175:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174492.877723:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174492.877725:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174492.877971:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174492.877973:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174492.878167:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174492.878169:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174492.878345:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174492.878346:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174497.997297:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174497.997304:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174497.998147:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174497.998152:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174497.998825:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174497.998828:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174497.999148:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174497.999150:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174497.999362:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174497.999364:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174503.117157:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174503.117163:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174503.117840:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174503.117842:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174503.118059:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174503.118061:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174503.118249:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174503.118251:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174503.118472:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174503.118474:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174508.237477:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174508.238126:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174508.238392:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174508.238575:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174508.238719:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174513.357968:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174513.357979:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174513.359717:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174513.359723:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174513.360259:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174513.360265:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174513.360900:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174513.360905:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174513.361583:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174513.361588:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174518.477249:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174518.477255:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174518.478241:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174518.478247:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174518.478698:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174518.478700:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174518.478968:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174518.478971:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174518.479387:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174518.479389:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174523.597622:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174523.597631:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174523.598666:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174523.598669:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174523.599012:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174523.599015:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174523.599460:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174523.599464:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786174523.599796:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174523.599800:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786174528.717061:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174528.717066:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174528.717744:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174528.717747:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174528.718027:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174528.718029:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174528.718247:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174528.718249:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174528.718448:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174528.718449:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174533.837627:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174533.837645:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174533.839414:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174533.839423:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174533.840238:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174533.840243:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174533.841061:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174533.841066:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174533.841699:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174533.841704:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174538.957288:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174538.957298:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174538.958239:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174538.958241:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174538.958515:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174538.958517:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174538.958763:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174538.958765:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174538.958972:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174538.958974:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174544.077286:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174544.077295:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174544.078287:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174544.078289:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174544.078507:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174544.078508:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174544.078743:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174544.078745:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174544.079011:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174544.079013:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174549.197131:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174549.197138:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174549.197819:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174549.197821:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174549.198090:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174549.198092:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174549.198333:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174549.198335:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174549.198628:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174549.198630:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174554.317204:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174554.317212:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174554.317837:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174554.317841:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174554.318202:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174554.318205:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174554.318572:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174554.318575:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174554.318933:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174554.318936:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174559.437669:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174559.437678:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786174559.438757:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174559.438760:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786174559.439207:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174559.439212:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786174559.439609:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174559.439612:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786174559.439991:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174559.439995:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786174564.557810:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174564.557822:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786174564.559046:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174564.559050:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786174564.559497:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174564.559501:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786174564.559949:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174564.559953:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786174564.560303:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174564.560306:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786174569.677672:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174569.677688:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786174569.679028:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174569.679032:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786174569.679557:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174569.679561:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786174569.680040:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174569.680044:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786174569.680621:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174569.680625:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786174574.797808:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174574.798656:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174574.799307:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174574.799677:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174574.800026:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174579.918179:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174579.918194:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174579.919829:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174579.919835:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174579.920279:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174579.920284:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174579.920841:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174579.920847:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174579.921307:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174579.921313:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174585.037337:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174585.037352:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174585.038655:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174585.038658:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174585.038920:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174585.038922:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174585.039183:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174585.039186:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174585.039571:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174585.039574:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174590.157750:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174590.157795:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786174590.158966:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174590.158969:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786174590.159351:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174590.159355:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786174590.159812:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174590.159815:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786174590.160175:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174590.160179:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786174595.277614:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174595.277637:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174595.279262:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174595.279274:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174595.279588:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174595.279589:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174595.279916:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174595.279919:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174595.280186:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174595.280188:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174600.397245:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174600.397251:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174600.398243:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174600.398249:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174600.398774:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174600.398776:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174600.399015:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174600.399017:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174600.399204:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174600.399205:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174605.517640:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174605.517654:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174605.518625:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174605.518630:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174605.519067:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174605.519071:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174605.519449:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174605.519452:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174605.519819:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174605.519822:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174610.637430:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174610.637443:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174610.638867:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174610.638869:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174610.639073:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174610.639074:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174610.639330:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174610.639332:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174610.639676:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174610.639679:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174615.757264:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174615.757275:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174615.758309:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174615.758311:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174615.758540:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174615.758542:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174615.758725:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174615.758727:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174615.758982:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174615.758984:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174620.877891:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174620.877906:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174620.879120:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174620.879122:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174620.879557:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174620.879559:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174620.879896:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174620.879899:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174620.880267:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174620.880281:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174625.997755:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174625.997761:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174625.999627:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174625.999638:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174626.000209:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174626.000213:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174626.000659:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174626.000661:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174626.000941:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174626.000943:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174631.117332:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174631.117341:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174631.118287:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174631.118290:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174631.118721:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174631.118725:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174631.119209:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174631.119212:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174631.119527:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174631.119530:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174636.237577:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174636.237587:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174636.238921:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174636.238924:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174636.239417:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174636.239420:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174636.239837:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174636.239840:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174636.240456:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174636.240459:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786174641.357302:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174641.357977:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174641.358728:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174641.359142:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174641.359252:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174646.477644:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174646.477655:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786174646.478565:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174646.478576:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786174646.479291:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174646.479293:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786174646.479632:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174646.479634:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786174646.479850:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174646.479852:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786174651.597443:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174651.597452:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174651.598437:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174651.598440:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174651.598777:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174651.598779:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174651.599104:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174651.599107:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786174651.599550:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174651.599564:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1786174656.717706:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174656.717720:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786174656.719073:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174656.719078:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786174656.720365:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174656.720366:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786174656.720703:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174656.720707:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786174656.721091:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174656.721093:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786174661.837324:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174661.837331:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174661.837999:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174661.838000:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174661.838231:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174661.838233:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174661.838463:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174661.838465:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174661.838700:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174661.838702:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174666.957436:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174666.957444:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174666.958350:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174666.958352:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174666.958726:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174666.958729:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174666.959033:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174666.959035:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174666.959316:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174666.959319:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174672.077139:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174672.077144:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174672.077926:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174672.077928:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174672.078194:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174672.078197:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174672.078449:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174672.078452:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174672.078780:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174672.078782:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174677.197555:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174677.197573:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174677.198635:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174677.198639:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174677.199030:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174677.199033:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174677.199354:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174677.199386:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174677.199791:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174677.199794:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174682.316937:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174682.316944:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174682.317790:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174682.317793:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174682.318119:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174682.318120:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174682.318388:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174682.318391:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174682.318759:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174682.318761:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174687.437260:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174687.437267:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1786174687.438200:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174687.438204:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1786174687.438717:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174687.438721:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1786174687.439082:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174687.439086:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1786174687.439686:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174687.439690:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 16 secs later) 00000800:00000200:2.1:1786174692.557780:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174692.557799:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1786174692.559887:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174692.559890:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1786174692.560431:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174692.560435:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1786174692.560882:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174692.560887:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1786174692.561438:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174692.561442:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1786174697.677716:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174697.677726:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786174697.678753:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174697.678757:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786174697.679161:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174697.679164:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786174697.679632:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174697.679636:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786174697.679988:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174697.679991:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786174702.796954:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174702.796957:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174702.797662:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174702.797665:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174702.797856:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174702.797857:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174702.798035:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174702.798037:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174702.798290:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174702.798292:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174707.917668:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174707.918429:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174707.918969:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174707.919307:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174707.919665:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174713.037474:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174713.037484:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174713.038639:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174713.038644:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174713.039234:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174713.039237:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174713.039604:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174713.039608:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174713.040015:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174713.040018:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174718.157054:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174718.157060:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174718.157624:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174718.157626:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174718.157818:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174718.157820:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174718.158024:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174718.158026:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174718.158206:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174718.158208:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174723.277816:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174723.277830:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174723.279061:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174723.279064:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174723.279496:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174723.279501:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174723.279946:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174723.279950:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174723.280290:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174723.280293:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174728.397130:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174728.397139:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174728.397779:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174728.397781:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174728.397978:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174728.397980:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174728.398210:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174728.398213:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174728.398438:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174728.398440:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786174733.517313:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174733.517325:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786174733.518319:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174733.518322:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786174733.518690:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174733.518695:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786174733.519059:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174733.519061:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786174733.519499:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174733.519502:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786174738.637831:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174738.637851:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174738.639464:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174738.639468:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174738.639968:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174738.639972:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174738.640458:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174738.640462:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174738.640928:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174738.640932:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786174743.757632:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174743.757646:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174743.759280:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174743.759282:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174743.759761:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174743.759763:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174743.760017:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174743.760020:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174743.760501:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174743.760505:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174748.877486:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174748.877500:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174748.878564:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174748.878567:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174748.878984:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174748.878987:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174748.879450:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174748.879455:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174748.879902:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174748.879906:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174753.997560:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174753.997571:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174753.998753:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174753.998759:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174753.999194:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174753.999198:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174753.999684:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174753.999688:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174754.000187:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174754.000190:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174759.117716:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174759.117731:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174759.119151:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174759.119154:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174759.119608:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174759.119612:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174759.120010:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174759.120013:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174759.120341:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174759.120344:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174764.237931:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174764.237948:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174764.239559:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174764.239563:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174764.240002:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174764.240006:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174764.240423:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174764.240427:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174764.240791:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174764.240794:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174769.357617:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174769.357623:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174769.358830:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174769.358834:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174769.359204:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174769.359207:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174769.359593:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174769.359596:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174769.360097:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174769.360101:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174774.477698:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174774.478496:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174774.479118:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174774.479496:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174774.479833:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174779.597844:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174779.597851:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174779.599240:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174779.599246:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174779.600499:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174779.600503:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174779.603187:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174779.603192:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174779.603624:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174779.603627:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174784.717350:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174784.717393:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174784.718178:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174784.718181:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174784.718519:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174784.718522:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174784.718822:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174784.718825:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174784.719180:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174784.719182:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174789.838008:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174789.838023:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786174789.839620:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174789.839625:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786174789.840009:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174789.840013:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786174789.840418:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174789.840422:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786174789.840938:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174789.840942:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786174794.957295:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174794.957302:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174794.958108:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174794.958110:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174794.958337:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174794.958339:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174794.958622:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174794.958624:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174794.960700:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174794.960704:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174800.077799:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174800.077815:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174800.078865:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174800.078869:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174800.079346:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174800.079350:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174800.079864:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174800.079868:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174800.080336:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174800.080341:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174805.197640:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174805.197668:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786174805.198953:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174805.198959:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786174805.199399:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174805.199407:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786174805.199852:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174805.199856:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786174805.200295:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174805.200299:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786174810.317186:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174810.317196:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786174810.318065:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174810.318069:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786174810.318575:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174810.318578:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786174810.318911:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174810.318913:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786174810.319139:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174810.319141:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786174815.437719:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174815.437744:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174815.439743:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174815.439763:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174815.440431:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174815.440435:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174815.440914:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174815.440918:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174815.441260:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174815.441264:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174820.557015:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174820.557021:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174820.557652:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174820.557653:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174820.557839:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174820.557840:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174820.558015:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174820.558016:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174820.558314:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174820.558316:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174825.677598:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174825.677609:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174825.678681:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174825.678684:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174825.679116:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174825.679120:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174825.679614:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174825.679618:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174825.679975:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174825.679979:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174830.797873:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174830.797892:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174830.799485:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174830.799489:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174830.799928:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174830.799931:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174830.800408:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174830.800412:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174830.800825:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174830.800829:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174835.917214:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174835.917220:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174835.918061:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174835.918063:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174835.918356:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174835.918357:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174835.918780:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174835.918782:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174835.919089:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174835.919092:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174841.037658:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174841.038486:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174841.038877:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174841.039134:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174841.039342:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174846.157966:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174846.157981:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174846.159697:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174846.159702:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174846.160312:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174846.160318:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174846.160911:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174846.160914:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174846.161625:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174846.161628:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174851.277435:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174851.277444:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174851.278329:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174851.278332:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174851.278621:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174851.278624:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174851.278922:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174851.278924:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174851.279238:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174851.279240:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174856.397984:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174856.398007:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174856.399252:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174856.399258:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174856.399807:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174856.399814:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174856.400266:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174856.400270:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174856.400667:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174856.400671:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174861.517976:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174861.517989:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174861.519324:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174861.519328:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174861.519720:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174861.519723:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174861.520096:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174861.520099:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174861.520476:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174861.520479:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786174866.637712:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174866.637720:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786174866.639050:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174866.639063:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786174866.639838:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174866.639840:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786174866.640138:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174866.640140:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786174866.640565:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174866.640569:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786174871.757104:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174871.757111:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174871.757847:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174871.757849:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174871.758089:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174871.758092:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174871.758321:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174871.758323:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174871.758592:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174871.758594:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786174876.876919:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174876.876923:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174876.877326:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174876.877328:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174876.877558:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174876.877560:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174876.877739:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174876.877741:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174876.877934:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174876.877935:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786174881.996925:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174881.996931:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174881.997494:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174881.997497:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174881.997681:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174881.997682:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174881.997880:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174881.997881:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174881.998052:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174881.998054:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786174887.117076:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174887.117081:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174887.117760:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174887.117762:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174887.118000:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174887.118003:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174887.118240:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174887.118243:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174887.118532:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174887.118534:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786174892.237727:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174892.237738:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174892.239098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174892.239101:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174892.239506:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174892.239510:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174892.239916:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174892.239920:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174892.240262:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174892.240265:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786174897.358097:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174897.358113:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174897.359673:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174897.359676:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174897.360005:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174897.360008:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174897.360469:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174897.360472:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174897.360855:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174897.360858:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786174902.477600:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174902.477612:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174902.478953:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174902.478956:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174902.479331:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174902.479334:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174902.479766:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174902.479769:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174902.480319:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174902.480322:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786174907.597522:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174907.598249:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174907.598893:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174907.599436:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174907.599869:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174912.717711:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174912.717718:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174912.719018:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174912.719031:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174912.719937:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174912.719940:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174912.720384:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174912.720388:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174912.720890:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174912.720894:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786174917.837686:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174917.837695:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174917.838816:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174917.838819:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174917.839300:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174917.839303:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174917.839741:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174917.839745:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174917.840173:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174917.840178:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786174922.957865:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174922.957876:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174922.958995:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174922.958999:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174922.959523:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174922.959527:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174922.959998:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174922.960001:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786174922.960519:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786174922.960523:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786174928.077281:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174928.077288:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174928.078211:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174928.078213:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174928.078619:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174928.078622:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174928.078968:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174928.078971:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174928.079390:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174928.079393:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786174933.197171:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174933.197177:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174933.197753:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174933.197755:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174933.198024:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174933.198025:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174933.198282:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174933.198284:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174933.198498:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174933.198500:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786174938.317590:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174938.317601:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786174938.318833:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174938.318836:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786174938.319247:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174938.319250:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786174938.319774:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174938.319777:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786174938.320141:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174938.320145:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786174943.437055:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174943.437062:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174943.437661:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174943.437664:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174943.437979:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174943.437982:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174943.438307:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174943.438310:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174943.438631:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174943.438634:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786174948.557632:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174948.557641:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174948.558871:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174948.558875:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174948.559269:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174948.559272:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174948.559651:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174948.559654:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174948.560144:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174948.560148:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786174953.677655:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174953.677674:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174953.678778:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174953.678781:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174953.679177:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174953.679180:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174953.679634:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174953.679637:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174953.679999:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174953.680002:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786174958.797638:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174958.797649:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174958.799161:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174958.799164:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174958.799538:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174958.799541:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174958.799914:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174958.799917:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174958.800289:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174958.800291:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786174963.917649:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174963.917659:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174963.918722:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174963.918726:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174963.919160:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174963.919163:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174963.919661:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174963.919665:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174963.920039:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174963.920042:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786174969.037709:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174969.037726:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174969.039404:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174969.039407:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174969.039844:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174969.039848:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174969.040248:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174969.040251:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174969.040835:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174969.040839:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786174974.157124:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174974.157554:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174974.157730:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174974.157916:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174974.158068:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174979.277675:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174979.277683:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174979.279561:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174979.279573:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174979.280444:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174979.280449:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174979.280995:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174979.280999:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174979.281520:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174979.281524:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786174984.397644:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174984.397656:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174984.399121:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174984.399125:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174984.399714:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174984.399718:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174984.400248:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174984.400254:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174984.401006:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174984.401010:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786174989.517603:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174989.517614:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786174989.518836:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174989.518839:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786174989.519211:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174989.519215:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786174989.519606:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174989.519610:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786174989.520016:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174989.520019:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786174994.637549:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174994.637563:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174994.638860:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174994.638863:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174994.639272:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174994.639275:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174994.639725:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174994.639728:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174994.640198:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174994.640221:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786174999.757192:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174999.757202:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174999.757907:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174999.757910:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174999.758219:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174999.758221:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174999.758468:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174999.758470:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786174999.758709:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786174999.758712:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175004.877744:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175004.877755:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175004.878919:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175004.878922:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175004.879352:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175004.879356:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175004.879837:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175004.879841:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175004.880268:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175004.880272:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175009.997295:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175009.997305:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175009.998013:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175009.998016:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175009.998282:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175009.998285:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175009.998609:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175009.998611:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175009.998816:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175009.998834:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175015.117738:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175015.117780:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175015.118981:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175015.118986:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175015.119365:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175015.119368:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175015.119976:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175015.119979:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175015.120332:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175015.120335:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175020.237613:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175020.237630:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175020.239092:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175020.239095:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175020.239562:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175020.239566:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175020.240036:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175020.240041:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175020.240536:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175020.240539:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175025.357321:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175025.357341:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175025.358101:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175025.358103:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175025.358296:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175025.358298:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175025.358544:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175025.358565:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175025.358834:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175025.358836:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175030.477600:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175030.477608:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786175030.478254:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175030.478257:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786175030.478592:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175030.478594:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786175030.478913:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175030.478916:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786175030.479210:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175030.479213:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786175035.597354:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175035.597385:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786175035.598328:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175035.598331:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786175035.598761:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175035.598763:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786175035.599038:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175035.599041:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786175035.599550:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175035.599553:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175040.717674:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175040.718527:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175040.719171:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175040.719608:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175040.719936:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175045.837796:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175045.837817:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175045.839460:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175045.839477:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175045.839965:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175045.839967:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175045.840295:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175045.840298:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175045.840608:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175045.840612:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175050.957448:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175050.957455:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175050.958248:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175050.958252:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175050.958663:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175050.958666:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175050.959020:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175050.959024:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175050.959436:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175050.959440:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175056.077669:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175056.077678:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175056.079053:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175056.079056:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175056.079480:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175056.079484:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175056.079862:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175056.079868:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175056.080496:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175056.080499:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175061.197427:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175061.197437:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175061.198267:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175061.198269:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175061.198674:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175061.198677:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175061.198955:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175061.198958:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175061.199254:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175061.199257:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175066.317753:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175066.317764:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175066.318852:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175066.318856:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175066.319294:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175066.319297:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175066.319787:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175066.319790:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175066.320235:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175066.320238:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175071.437665:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175071.437745:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175071.439268:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175071.439277:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175071.439712:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175071.439714:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175071.440030:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175071.440032:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175071.440479:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175071.440481:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175076.557572:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175076.557578:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175076.559067:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175076.559070:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175076.559530:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175076.559533:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175076.559944:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175076.559950:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175076.560614:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175076.560618:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175081.677769:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175081.677841:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175081.679282:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175081.679284:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175081.679638:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175081.679641:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175081.679903:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175081.679905:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175081.680194:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175081.680196:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175086.797596:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175086.797606:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175086.799223:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175086.799237:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175086.799597:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175086.799598:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175086.799857:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175086.799859:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175086.800040:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175086.800042:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175091.917785:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175091.917793:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175091.919052:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175091.919056:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175091.919494:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175091.919498:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175091.919952:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175091.919955:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175091.920312:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175091.920315:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175097.037157:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175097.037164:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175097.037944:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175097.037945:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175097.038156:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175097.038158:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175097.038438:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175097.038440:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175097.038791:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175097.038794:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175102.157666:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175102.157673:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175102.159061:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175102.159071:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175102.159646:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175102.159650:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175102.160075:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175102.160079:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175102.160664:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175102.160668:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175107.277796:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175107.278527:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175107.279265:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175107.279613:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175107.279874:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175112.397770:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175112.397776:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175112.398907:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175112.398910:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175112.399131:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175112.399133:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175112.399344:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175112.399346:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175112.399599:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175112.399601:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175117.517637:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175117.517661:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786175117.519043:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175117.519047:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786175117.519507:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175117.519510:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786175117.519918:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175117.519921:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786175117.520359:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175117.520362:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1786175122.637163:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175122.637169:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175122.637919:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175122.637922:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175122.638237:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175122.638240:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175122.638715:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175122.638718:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175122.639322:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175122.639326:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175127.757877:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175127.757892:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175127.759656:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175127.759661:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175127.760112:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175127.760116:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175127.760635:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175127.760641:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175127.761245:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175127.761248:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175132.877641:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175132.877652:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175132.878921:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175132.878925:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175132.879342:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175132.879345:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175132.879815:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175132.879828:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175132.880172:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175132.880186:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175137.997156:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175137.997161:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175137.997881:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175137.997883:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175137.998081:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175137.998083:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175137.998344:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175137.998346:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175137.998560:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175137.998562:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175143.117167:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175143.117175:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175143.117935:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175143.117938:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175143.118308:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175143.118309:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175143.118578:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175143.118582:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175143.118816:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175143.118818:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175148.236954:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175148.236960:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175148.237526:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175148.237528:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175148.237723:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175148.237724:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175148.237908:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175148.237910:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175148.238081:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175148.238082:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175153.357054:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175153.357063:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175153.357687:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175153.357690:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175153.358016:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175153.358019:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175153.358347:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175153.358350:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175153.358677:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175153.358680:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175158.476993:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175158.476997:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1786175158.477841:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175158.477845:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1786175158.478156:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175158.478157:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1786175158.478355:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175158.478383:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1786175158.478754:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175158.478756:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786175163.597473:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175163.597484:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786175163.598902:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175163.598910:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786175163.599422:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175163.599424:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786175163.599704:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175163.599705:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786175163.599888:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175163.599890:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786175168.717653:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175168.717676:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786175168.719456:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175168.719484:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786175168.720210:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175168.720213:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786175168.720740:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175168.720743:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786175168.721215:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175168.721218:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175173.838069:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175173.839047:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175173.839401:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175173.839731:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175173.840691:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175178.957890:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175178.957902:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175178.959523:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175178.959527:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175178.959978:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175178.959982:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175178.960347:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175178.960352:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175178.960794:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175178.960797:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175184.077113:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175184.077120:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175184.077735:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175184.077737:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175184.077976:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175184.077978:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175184.078203:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175184.078205:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175184.078472:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175184.078474:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175189.197111:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175189.197118:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175189.197827:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175189.197829:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175189.198042:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175189.198044:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175189.198284:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175189.198287:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175189.198568:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175189.198570:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175194.317663:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175194.317672:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175194.319047:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175194.319051:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175194.319463:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175194.319466:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175194.319910:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175194.319913:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175194.320450:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175194.320454:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786175199.437058:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175199.437063:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175199.437632:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175199.437634:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175199.437872:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175199.437874:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175199.438090:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175199.438092:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175199.438299:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175199.438301:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175204.557887:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175204.557895:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175204.559170:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175204.559175:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175204.559834:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175204.559839:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175204.560214:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175204.560217:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175204.560809:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175204.560815:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175209.677832:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175209.677844:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175209.679011:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175209.679015:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175209.679410:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175209.679413:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175209.679792:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175209.679795:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175209.680161:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175209.680164:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175214.798026:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175214.798038:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175214.799752:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175214.799755:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175214.800071:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175214.800074:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175214.800352:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175214.800355:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175214.800869:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175214.800873:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175219.917485:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175219.917497:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175219.918673:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175219.918679:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175219.919522:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175219.919526:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175219.919973:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175219.919976:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175219.920315:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175219.920318:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175225.037826:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175225.037842:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175225.039560:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175225.039564:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175225.039999:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175225.040005:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175225.040508:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175225.040511:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175225.041214:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175225.041218:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175230.157530:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175230.157543:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175230.158892:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175230.158896:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175230.159273:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175230.159277:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175230.159756:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175230.159762:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175230.160155:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175230.160159:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175235.277803:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175235.277814:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175235.279490:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175235.279494:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175235.279878:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175235.279882:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175235.280333:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175235.280336:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175235.280754:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175235.280758:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175240.397777:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175240.398620:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175240.399486:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175240.399831:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175240.400138:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175245.517789:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175245.517796:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175245.519976:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175245.519981:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175245.520645:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175245.520648:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175245.521158:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175245.521163:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175245.521767:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175245.521770:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175250.637844:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175250.637853:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175250.639745:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175250.639754:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175250.640539:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175250.640543:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175250.640988:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175250.640991:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175250.641386:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175250.641390:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175255.757526:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175255.757538:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175255.758845:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175255.758848:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175255.759232:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175255.759235:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175255.759622:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175255.759626:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175255.760132:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175255.760136:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175260.877632:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175260.877656:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786175260.878443:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175260.878446:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786175260.878819:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175260.878822:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786175260.879143:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175260.879145:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786175260.879420:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175260.879423:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786175265.997668:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175265.997684:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175265.998895:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175265.998899:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175265.999392:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175265.999396:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175265.999826:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175265.999830:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175266.000265:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175266.000271:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175271.118101:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175271.118121:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175271.119843:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175271.119847:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175271.120274:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175271.120277:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175271.120768:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175271.120771:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175271.121173:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175271.121177:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175276.237273:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175276.237281:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786175276.238540:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175276.238545:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786175276.238875:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175276.238878:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786175276.239256:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175276.239258:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786175276.239779:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175276.239782:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786175281.357750:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175281.357765:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786175281.359018:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175281.359024:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786175281.359469:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175281.359473:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786175281.359974:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175281.359979:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786175281.360433:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175281.360437:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786175286.477497:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175286.477504:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175286.478753:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175286.478756:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175286.479167:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175286.479170:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175286.479614:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175286.479618:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175286.480136:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175286.480140:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175291.597032:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175291.597038:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175291.597801:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175291.597803:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175291.598024:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175291.598025:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175291.598241:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175291.598242:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175291.598487:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175291.598489:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175296.717660:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175296.717671:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175296.718823:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175296.718826:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175296.719182:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175296.719185:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175296.719559:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175296.719562:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175296.719933:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175296.719937:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175301.837344:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175301.837352:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175301.838370:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175301.838373:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175301.838801:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175301.838803:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175301.838996:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175301.838998:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175301.839214:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175301.839216:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175306.956966:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175306.957340:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175306.957707:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175306.957876:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175306.958020:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175312.077159:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175312.077165:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175312.077951:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175312.077954:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175312.078288:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175312.078292:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175312.078837:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175312.078840:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175312.079159:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175312.079162:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175317.197111:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175317.197118:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175317.197767:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175317.197770:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175317.198098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175317.198101:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175317.198437:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175317.198440:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175317.198757:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175317.198760:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175322.317628:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175322.317637:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175322.319013:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175322.319019:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175322.319696:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175322.319701:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175322.320308:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175322.320314:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175322.321247:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175322.321252:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175327.437155:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175327.437165:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786175327.437928:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175327.437930:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786175327.438239:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175327.438242:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786175327.438681:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175327.438685:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786175327.439023:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175327.439026:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786175332.557560:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175332.557573:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175332.559069:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175332.559072:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175332.559562:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175332.559565:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175332.560233:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175332.560236:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175332.560617:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175332.560619:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175337.677034:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175337.677041:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175337.677748:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175337.677749:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175337.677970:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175337.677972:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175337.678216:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175337.678218:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175337.678556:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175337.678558:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175342.797279:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175342.797282:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175342.797991:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175342.797994:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175342.798238:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175342.798240:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175342.798557:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175342.798560:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175342.798873:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175342.798875:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175347.917716:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175347.917733:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175347.919106:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175347.919109:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175347.919566:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175347.919569:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175347.919955:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175347.919959:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175347.920352:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175347.920355:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175353.037606:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175353.037617:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175353.038599:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175353.038602:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175353.038943:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175353.038946:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175353.039454:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175353.039457:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175353.039822:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175353.039826:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175358.157237:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175358.157242:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175358.158223:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175358.158226:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175358.158691:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175358.158694:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175358.158953:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175358.158955:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175358.159483:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175358.159486:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175363.277214:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175363.277222:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175363.277960:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175363.277962:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175363.278308:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175363.278310:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175363.278598:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175363.278600:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175363.278816:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175363.278818:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175368.397266:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175368.397271:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175368.398034:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175368.398053:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175368.398375:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175368.398378:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175368.398752:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175368.398754:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175368.399045:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175368.399048:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175373.517116:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175373.517587:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175373.517784:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175373.517973:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175373.518116:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175378.637185:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175378.637192:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175378.638068:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175378.638071:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175378.638430:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175378.638433:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175378.638734:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175378.638736:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175378.639015:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175378.639017:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175383.757030:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175383.757054:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175383.757693:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175383.757695:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175383.757925:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175383.757927:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175383.758131:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175383.758133:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175383.758413:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175383.758415:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175388.877094:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175388.877099:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175388.877840:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175388.877842:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175388.878045:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175388.878047:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175388.878264:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175388.878266:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175388.878491:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175388.878493:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175393.997757:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175393.997783:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175393.999042:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175393.999044:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175393.999247:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175393.999249:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175393.999462:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175393.999464:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175393.999651:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175393.999653:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175399.117040:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175399.117047:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175399.117741:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175399.117744:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175399.117940:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175399.117942:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175399.118155:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175399.118157:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175399.118331:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175399.118333:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175404.237206:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175404.237220:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786175404.238384:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175404.238386:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786175404.238587:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175404.238589:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786175404.238797:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175404.238799:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786175404.239115:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175404.239118:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786175409.357029:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175409.357036:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786175409.357742:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175409.357744:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786175409.357938:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175409.357939:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786175409.358117:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175409.358118:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786175409.358377:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175409.358379:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786175414.477033:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175414.477037:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175414.477748:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175414.477750:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175414.477974:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175414.477975:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175414.478188:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175414.478190:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175414.478505:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175414.478507:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175419.597621:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175419.597628:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175419.598572:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175419.598577:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175419.600099:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175419.600111:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175419.601048:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175419.601053:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175419.601722:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175419.601727:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175424.717219:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175424.717224:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175424.717992:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175424.717996:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175424.718499:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175424.718501:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175424.718795:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175424.718797:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175424.719061:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175424.719063:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175429.837757:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175429.837765:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175429.839198:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175429.839202:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175429.839746:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175429.839750:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175429.840133:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175429.840139:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175429.840977:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175429.840981:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175434.957458:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175434.957466:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175434.958631:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175434.958634:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175434.959021:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175434.959023:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175434.959284:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175434.959286:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175434.959686:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175434.959690:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175440.076974:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175440.077399:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175440.077763:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175440.077949:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175440.078122:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175445.197298:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175445.197302:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175445.198184:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175445.198191:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175445.198621:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175445.198623:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175445.198983:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175445.198986:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175445.199346:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175445.199348:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175450.317028:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175450.317034:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175450.317721:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175450.317722:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175450.317919:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175450.317921:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175450.318095:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175450.318097:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175450.318322:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175450.318324:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175455.437674:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175455.437703:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175455.438990:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175455.438994:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175455.439525:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175455.439529:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175455.440002:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175455.440006:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175455.440507:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175455.440511:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175460.557218:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175460.557228:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175460.558151:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175460.558158:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175460.558710:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175460.558714:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175460.559066:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175460.559068:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175460.559436:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175460.559439:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175465.677172:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175465.677180:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175465.677967:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175465.677969:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175465.678274:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175465.678276:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175465.678527:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175465.678529:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175465.678811:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175465.678813:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175470.797480:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175470.797493:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175470.798432:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175470.798434:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175470.798654:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175470.798656:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175470.798850:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175470.798852:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175470.799093:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175470.799095:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175475.917533:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175475.917542:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175475.918485:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175475.918487:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175475.918789:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175475.918792:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175475.919105:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175475.919108:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175475.919517:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175475.919520:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175481.037720:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175481.037737:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175481.039031:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175481.039034:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175481.039487:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175481.039491:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175481.039869:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175481.039872:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175481.040277:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175481.040280:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175486.157268:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175486.157274:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175486.158117:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175486.158119:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175486.158374:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175486.158376:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175486.158682:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175486.158685:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175486.158969:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175486.158971:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175491.277080:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175491.277087:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175491.277706:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175491.277708:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175491.277921:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175491.277923:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175491.278105:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175491.278106:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175491.278465:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175491.278467:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175496.397252:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175496.397264:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175496.398286:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175496.398287:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175496.398503:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175496.398505:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175496.398733:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175496.398735:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175496.399009:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175496.399011:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175501.517184:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175501.517194:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175501.518173:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175501.518181:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175501.518933:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175501.518934:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175501.519216:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175501.519218:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175501.519437:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175501.519439:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175506.637631:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175506.638472:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175506.639044:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175506.639324:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175506.639575:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175511.757196:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175511.757204:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175511.758024:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175511.758029:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175511.758845:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175511.758850:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175511.759218:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175511.759221:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175511.759493:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175511.759496:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175516.877511:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175516.877517:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175516.878662:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175516.878664:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175516.878925:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175516.878927:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175516.879255:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175516.879258:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175516.879519:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175516.879521:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175521.997424:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175521.997434:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175521.998125:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175521.998129:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175521.998496:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175521.998498:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175521.998854:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175521.998856:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175521.999146:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175521.999150:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175527.117013:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175527.117018:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786175527.117743:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175527.117745:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786175527.117975:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175527.117977:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786175527.118161:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175527.118163:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786175527.118530:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175527.118532:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786175532.237529:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175532.237537:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175532.239005:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175532.239007:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175532.239277:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175532.239280:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175532.239639:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175532.239641:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175532.240002:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175532.240005:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175537.357289:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175537.357294:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175537.358170:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175537.358172:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175537.358536:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175537.358538:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175537.358850:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175537.358852:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175537.359244:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175537.359246:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786175542.477351:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175542.477408:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175542.478307:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175542.478310:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175542.478666:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175542.478668:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175542.479011:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175542.479013:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175542.479503:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175542.479506:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175547.597370:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175547.597406:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175547.598698:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175547.598710:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175547.599128:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175547.599129:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175547.599483:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175547.599485:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175547.599667:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175547.599668:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175552.717108:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175552.717114:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175552.717844:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175552.717846:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175552.718117:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175552.718119:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175552.718438:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175552.718440:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175552.718702:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175552.718704:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175557.836958:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175557.836961:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175557.837403:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175557.837405:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175557.837656:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175557.837658:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175557.837868:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175557.837870:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175557.838043:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175557.838045:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175562.957284:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175562.957294:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175562.958230:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175562.958232:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175562.958459:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175562.958461:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175562.958691:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175562.958693:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175562.958882:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175562.958884:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175568.077251:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175568.077260:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175568.078286:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175568.078292:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175568.078694:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175568.078696:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175568.079030:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175568.079033:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175568.079323:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175568.079325:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175573.197766:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175573.198657:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175573.198984:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175573.199331:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175573.199672:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175578.317687:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175578.317692:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175578.318682:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175578.318685:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175578.318969:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175578.318971:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175578.319807:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175578.319810:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175578.320115:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175578.320117:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175583.437267:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175583.437272:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1786175583.437891:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175583.437892:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1786175583.438113:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175583.438115:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1786175583.438298:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175583.438300:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1786175583.438506:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175583.438508:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1786175588.557521:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175588.557531:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175588.558554:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175588.558558:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175588.558949:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175588.558952:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175588.559282:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175588.559285:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175588.559755:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175588.559758:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786175593.677295:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175593.677300:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175593.678453:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175593.678458:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175593.678867:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175593.678897:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175593.679256:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175593.679259:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175593.679550:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175593.679552:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786175598.797772:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175598.797814:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175598.799340:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175598.799344:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175598.799753:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175598.799756:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175598.800203:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175598.800206:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786175598.800601:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175598.800604:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175603.917900:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175603.917911:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175603.919648:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175603.919652:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175603.920169:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175603.920172:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175603.920697:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175603.920702:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175603.921423:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175603.921440:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175609.037957:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175609.037973:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175609.039615:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175609.039619:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175609.040053:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175609.040057:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175609.040495:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175609.040499:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175609.041052:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175609.041056:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000400:00000200:3.0:1786175614.156686:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175614.156689:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175614.156691:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175614.156696:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.156698:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175614.156700:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.156701:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175614.156703:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175614.156703:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175614.156704:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175614.156706:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175614.156707:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175614.156709:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175614.156711:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.156712:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175614.156724:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.156730:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175614.156736:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175614.156745:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175614.156754:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175614.156765:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175614.156777:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175614.156786:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.156790:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175614.156795:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175614.156803:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.156809:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175614.156824:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786175614.156829:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175614.156831:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175614.156838:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.156850:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175614.156857:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175614.156860:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175614.156866:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786175614.156872:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175614.156881:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235389568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175614.156896:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175614.156899:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175614.156906:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235389568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175614.156914:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175614.156919:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175614.156921:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175614.156922:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175614.156924:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175614.156926:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175614.156928:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10288 00000400:00000200:3.0:1786175614.156932:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175614.156937:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175614.156943:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175614.156952:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.156955:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175614.156961:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.156963:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175614.156966:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175614.156969:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175614.156973:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175614.156976:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175614.156981:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175614.156989:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175614.156998:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157008:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175614.157014:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157020:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175614.157026:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175614.157034:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157043:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175614.157044:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175614.157047:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175614.157057:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.157068:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175614.157071:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175614.157073:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.157075:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175614.157084:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786175614.157089:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175614.157095:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175614.157099:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.157107:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175614.157110:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175614.157112:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175614.157120:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786175614.157129:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175614.157139:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235389696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175614.157179:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175614.157183:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175614.157189:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235389696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175614.157231:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175614.157240:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175614.157245:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175614.157254:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175614.157257:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175614.157266:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175614.157276:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10288 00000400:00000200:3.0:1786175614.157289:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:1.1:1786175614.157294:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:3.0:1786175614.157298:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175614.157302:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1786175614.157302:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000040:3.0:1786175614.157311:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157319:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175614.157326:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157328:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175614.157332:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175614.157334:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175614.157337:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175614.157342:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175614.157355:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175614.157409:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175614.157422:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157433:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175614.157437:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157446:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175614.157452:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175614.157464:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157475:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175614.157484:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175614.157488:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175614.157498:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.157501:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175614.157506:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175614.157517:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.157527:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175614.157541:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786175614.157551:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175614.157555:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175614.157557:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.157563:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175614.157564:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175614.157566:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175614.157570:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786175614.157573:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175614.157580:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235389824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175614.157589:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175614.157592:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175614.157595:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235389824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175614.157605:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175614.157609:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175614.157622:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175614.157625:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175614.157644:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175614.157648:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175614.157655:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10288 00000400:00000200:3.0:1786175614.157669:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175614.157680:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175614.157688:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175614.157692:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157698:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175614.157705:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157720:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175614.157730:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175614.157736:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175614.157745:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175614.157753:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175614.157761:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175614.157773:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175614.157783:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157791:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175614.157801:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157810:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175614.157820:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175614.157832:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175614.157835:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175614.157837:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175614.157844:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175614.157855:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.157862:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175614.157865:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175614.157870:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.157876:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175614.157883:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786175614.157887:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175614.157891:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175614.157894:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175614.157896:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175614.157902:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175614.157905:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175614.157912:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786175614.157916:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175614.157932:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235389952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175614.157959:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175614.157965:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175614.157970:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235389952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175614.158016:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175614.158028:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10298) 00000800:00000200:1.1:1786175614.158195:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175614.158196:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175614.158471:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175614.158473:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175614.158712:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175614.158714:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175614.158929:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175614.158931:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175619.277958:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175619.277973:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175619.279710:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175619.279714:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175619.280236:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175619.280239:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175619.280670:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175619.280673:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175619.281315:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175619.281319:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.0:1786175622.796533:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:3.0:1786175622.796546:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:3.0:1786175622.796605:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:2.1:1786175624.399750:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175624.399764:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175624.400777:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175624.400781:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175624.401244:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175624.401248:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175624.401770:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175624.401773:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175624.402235:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175624.402239:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175629.517764:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175629.517775:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786175629.518830:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175629.518833:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786175629.519397:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175629.519400:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786175629.519910:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175629.519916:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786175629.520471:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175629.520477:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786175634.637969:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175634.637980:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175634.639705:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175634.639711:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175634.640240:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175634.640243:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175634.640721:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175634.640727:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175634.641110:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175634.641114:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000100:00000001:3.0:1786175639.244738:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175639.244748:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175639.244752:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175639.244761:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175639.244766:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175639.244775:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175639.244780:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175639.244784:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175639.244789:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175639.757663:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175639.758626:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175639.758939:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175639.759253:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175639.759614:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786175640.268498:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175640.268510:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175640.268512:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175640.268519:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175640.268523:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175640.268527:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175640.268530:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175640.268533:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175640.268536:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175641.292507:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175641.292522:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175641.292526:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175641.292535:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175641.292541:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175641.292547:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175641.292553:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175641.292559:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175641.292565:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175642.316496:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175642.316508:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175642.316511:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175642.316518:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175642.316524:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175642.316528:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175642.316543:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175642.316547:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175642.316552:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1786175642.828804:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:3.0:1786175642.828891:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1786175642.828911:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175642.828917:0:2381:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:3.0:1786175642.828926:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175642.828932:0:2381:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1786175642.828939:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175642.828946:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:3.0:1786175642.828956:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175642.828962:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175642.828973:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ea18 00000400:00000010:3.0:1786175642.828978:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ea18. 00000100:00000001:3.0:1786175642.828987:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175642.829036:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31737b80 x1872930235392640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786175664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175642.829086:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175642.829096:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235392640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786175664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175642.829113:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175642.829117:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175642.829139:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175642.829144:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:3.0:1786175642.829152:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175642.829161:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175642.829170:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:3.0:1786175642.829174:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:3.0:1786175642.829180:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175642.829194:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735c00 x1872930235392768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175642.829225:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175642.829243:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235392768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175642.829308:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175642.829315:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175642.829337:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175642.829349:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:3.0:1786175642.829541:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175642.829560:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175642.829578:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e110 00000400:00000010:3.0:1786175642.829586:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e110. 00000100:00000001:3.0:1786175642.829599:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175642.829614:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31734380 x1872930235392896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175642.829662:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175642.829670:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734380 x1872930235392896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175642.829701:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175642.829710:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175642.829734:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175642.829745:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:3.0:1786175642.829758:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175642.829778:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175642.829789:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e330 00000400:00000010:3.0:1786175642.829796:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e330. 00000100:00000001:3.0:1786175642.829806:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175642.829964:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31736300 x1872930235393024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175642.830016:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175642.830032:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736300 x1872930235393024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175642.830062:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175642.830074:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175642.830099:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175642.830112:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:3.0:1786175642.830126:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175642.830150:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175642.830167:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e880 00000400:00000010:3.0:1786175642.830177:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e880. 00000100:00000001:3.0:1786175642.830191:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175642.830238:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31734000 x1872930235393152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175642.830291:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175642.830311:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734000 x1872930235393152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175642.830347:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175642.830452:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1786175643.340488:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175643.340500:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175643.340502:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175643.340508:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175643.340514:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175643.340520:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175643.340524:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175643.340526:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175643.340529:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175644.364459:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175644.364466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175644.364468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175644.364474:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175644.364478:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175644.364481:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175644.364483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175644.364486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175644.364488:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175644.877827:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175644.877841:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175644.879299:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175644.879304:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175644.879909:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175644.879912:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175644.880439:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175644.880442:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175644.880789:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175644.880792:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786175645.388437:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175645.388447:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175645.388450:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175645.388456:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175645.388462:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175645.388469:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175645.388473:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175645.388476:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175645.388481:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175646.412457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175646.412469:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175646.412472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175646.412480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175646.412486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175646.412491:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175646.412496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175646.412500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175646.412506:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175649.997191:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175649.997199:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175649.998027:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175649.998029:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175649.998348:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175649.998351:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175649.998674:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175649.998677:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175649.999082:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175649.999084:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175655.117828:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175655.117836:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175655.118994:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175655.118998:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175655.119501:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175655.119505:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175655.119893:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175655.119897:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175655.120261:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175655.120265:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175660.237577:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175660.237591:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175660.239229:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175660.239232:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175660.239544:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175660.239547:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175660.239888:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175660.239891:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175660.240345:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175660.240348:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175665.357093:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175665.357116:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175665.357957:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175665.357959:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175665.358220:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175665.358222:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175665.358449:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175665.358450:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175665.358658:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175665.358660:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00080000:3.0:1786175670.476538:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175670.476563:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175670.476567:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175670.476572:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175670.476579:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175670.476588:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10344 00000400:00000200:3.0:1786175670.476600:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175670.476608:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175670.476613:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175670.476618:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.476623:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175670.476626:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.476630:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175670.476635:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175670.476638:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175670.476642:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175670.476646:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175670.476648:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175670.476652:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175670.476655:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.476658:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175670.476661:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.476663:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175670.476668:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175670.476672:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175670.476675:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175670.476679:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175670.476683:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175670.476687:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.476689:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175670.476692:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175670.476703:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786175670.476708:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175670.476713:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175670.476715:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.476719:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175670.476723:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175670.476729:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175670.476740:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786175670.476748:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175670.476761:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235396480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175670.476784:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175670.476790:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175670.476796:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235396480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175670.476826:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175670.476832:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175670.476843:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175670.476845:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175670.476848:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175670.476851:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175670.476855:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10344 00000400:00000200:3.0:1786175670.476859:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175670.476864:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175670.476867:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175670.476871:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.476875:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175670.476879:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.476882:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175670.476885:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175670.476887:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175670.476890:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175670.476892:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175670.476896:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175670.476901:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175670.476909:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.476917:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175670.476921:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.476928:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175670.476933:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175670.476942:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175670.476948:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175670.476956:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175670.476966:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175670.476971:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.476973:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175670.476988:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175670.476996:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.477003:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175670.477010:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786175670.477018:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175670.477028:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175670.477034:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.477040:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175670.477044:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175670.477050:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175670.477058:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786175670.477067:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175670.477082:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235396608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175670.477133:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175670.477141:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175670.477153:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235396608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175670.477175:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175670.477183:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175670.477203:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175670.477210:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175670.477233:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175670.477248:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175670.477263:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10344 00000400:00000200:3.0:1786175670.477283:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175670.477304:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175670.477310:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175670.477319:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.477326:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175670.477337:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.477345:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175670.477354:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175670.477423:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175670.477427:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175670.477432:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175670.477438:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175670.477449:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175670.477464:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.477469:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175670.477479:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.477487:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175670.477495:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175670.477508:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175670.477515:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175670.477520:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175670.477526:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175670.477533:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.477535:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175670.477542:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175670.477562:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.477569:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175670.477576:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786175670.477583:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000800:00000200:1.1:1786175670.477590:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786175670.477591:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175670.477595:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.477604:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1786175670.477605:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 02000000:00000001:3.0:1786175670.477607:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175670.477613:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175670.477622:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786175670.477630:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175670.477644:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235396736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175670.477663:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175670.477671:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175670.477683:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235396736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175670.477705:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175670.477715:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175670.477725:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175670.477727:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175670.477729:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175670.477733:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175670.477740:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10344 00000400:00000200:3.0:1786175670.477757:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175670.477767:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175670.477774:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175670.477784:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.477792:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175670.477801:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.477810:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175670.477816:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175670.477821:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175670.477825:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175670.477828:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175670.477832:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175670.477839:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175670.477853:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.477858:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175670.477867:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.477871:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175670.477875:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175670.477885:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175670.477889:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175670.477891:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175670.477896:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175670.477906:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.477909:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175670.477913:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175670.477917:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.477927:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175670.477934:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786175670.477942:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175670.477946:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175670.477952:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.477957:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175670.477960:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175670.477966:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175670.477975:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786175670.477991:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175670.478003:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235396864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175670.478053:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175670.478058:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175670.478070:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235396864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175670.478122:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175670.478130:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175670.478139:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175670.478144:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175670.478149:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175670.478162:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175670.478173:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10344 00000400:00000200:3.0:1786175670.478193:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175670.478206:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175670.478215:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175670.478232:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.478242:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175670.478255:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.478268:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175670.478280:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175670.478285:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175670.478301:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175670.478307:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175670.478314:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175670.478321:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175670.478333:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.478340:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175670.478348:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175670.478392:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175670.478404:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175670.478415:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175670.478428:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175670.478436:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175670.478444:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175670.478453:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.478460:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175670.478468:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175670.478477:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.478483:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175670.478495:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786175670.478501:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175670.478508:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175670.478514:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175670.478520:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175670.478524:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175670.478530:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175670.478541:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786175670.478559:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175670.478572:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235396992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175670.478590:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175670.478597:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175670.478606:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235396992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175670.478626:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175670.478634:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10355) 00000800:00000200:1.1:1786175670.478837:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175670.478840:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175670.479472:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175670.479477:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175670.479955:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175670.479960:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175670.480440:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175670.480444:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175675.596990:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175675.596996:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175675.597597:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175675.597599:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175675.597779:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175675.597781:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175675.597956:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175675.597958:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175675.598215:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175675.598217:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175680.717845:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175680.717860:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175680.719494:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175680.719497:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175680.719850:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175680.719854:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175680.720207:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175680.720210:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175680.720735:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175680.720738:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1786175682.252440:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175682.252447:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175682.252448:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175682.252453:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175682.252457:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175682.252460:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175682.252462:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175682.252465:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175682.252467:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175683.276444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175683.276451:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175683.276452:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175683.276457:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175683.276460:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175683.276464:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175683.276466:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175683.276468:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175683.276469:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175684.300469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175684.300481:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175684.300485:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175684.300493:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175684.300499:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175684.300505:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175684.300509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175684.300513:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175684.300518:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.324436:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175685.324445:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175685.324447:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175685.324454:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175685.324458:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175685.324462:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175685.324465:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175685.324468:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175685.324471:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175685.836515:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175685.836531:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175685.836535:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175685.836539:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175685.836545:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175685.836554:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10360 00000400:00000200:3.0:1786175685.836581:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175685.836590:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175685.836594:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175685.836599:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.836603:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175685.836627:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.836631:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175685.836637:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175685.836640:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175685.836643:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175685.836647:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175685.836648:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175685.836651:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175685.836654:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.836655:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175685.836657:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.836658:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175685.836661:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175685.836663:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175685.836665:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175685.836667:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175685.836682:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175685.836684:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.836686:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175685.836689:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175685.836698:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786175685.836701:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175685.836704:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175685.836706:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.836709:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175685.836712:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175685.836717:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175685.836726:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786175685.836734:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175685.836744:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235398400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175685.836753:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175685.836758:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175685.836762:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235398400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175685.836786:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175685.836789:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175685.836792:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175685.836793:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175685.836794:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175685.836796:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175685.836798:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10360 00000400:00000200:3.0:1786175685.836801:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175685.836804:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175685.836806:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175685.836808:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.836809:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175685.836811:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.836813:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175685.836814:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175685.836816:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175685.836817:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175685.836835:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175685.836838:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175685.836840:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175685.836843:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.836845:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175685.836849:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.836854:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175685.836859:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175685.836866:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175685.836872:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175685.836878:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175685.836886:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175685.836895:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.836901:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175685.836909:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175685.836916:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.836921:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175685.836930:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786175685.836956:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175685.836963:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175685.836966:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.836968:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175685.836970:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175685.836976:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175685.837009:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786175685.837015:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175685.837023:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235398528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175685.837041:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175685.837045:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175685.837053:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235398528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175685.837067:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175685.837071:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175685.837078:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175685.837080:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175685.837081:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175685.837084:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175685.837086:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10360 00000400:00000200:3.0:1786175685.837094:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175685.837100:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175685.837102:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175685.837109:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837114:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175685.837118:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837122:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175685.837124:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175685.837128:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175685.837129:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175685.837133:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175685.837135:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175685.837138:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175685.837142:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837145:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175685.837147:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837156:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175685.837160:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175685.837165:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837169:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175685.837171:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175685.837174:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175685.837175:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.837177:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175685.837196:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175685.837201:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.837202:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175685.837218:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786175685.837224:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175685.837229:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175685.837233:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.837235:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175685.837237:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175685.837240:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175685.837246:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786175685.837254:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175685.837261:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235398656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175685.837277:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175685.837282:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175685.837303:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235398656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175685.837316:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175685.837320:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175685.837329:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175685.837332:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175685.837335:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175685.837342:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175685.837346:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10360 00000400:00000200:3.0:1786175685.837353:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175685.837358:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175685.837360:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175685.837365:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837367:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175685.837369:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837372:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:2.1:1786175685.837394:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175685.837398:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000020:00000001:3.0:1786175685.837407:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175685.837411:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175685.837414:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175685.837417:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175685.837423:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175685.837429:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175685.837437:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837440:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175685.837448:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837454:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175685.837480:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175685.837493:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837502:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175685.837512:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175685.837521:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175685.837531:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.837550:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175685.837554:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175685.837561:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.837564:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175685.837569:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786175685.837574:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175685.837580:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175685.837582:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.837585:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175685.837590:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175685.837601:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175685.837608:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786175685.837614:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175685.837626:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235398784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175685.837643:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175685.837647:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175685.837658:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235398784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175685.837679:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175685.837683:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175685.837690:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175685.837693:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175685.837697:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175685.837704:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175685.837708:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10360 00000400:00000200:3.0:1786175685.837717:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175685.837725:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175685.837729:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175685.837734:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837738:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175685.837743:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837745:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175685.837749:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175685.837750:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175685.837752:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175685.837754:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175685.837756:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175685.837760:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175685.837766:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837770:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175685.837775:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837779:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175685.837783:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175685.837789:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175685.837791:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175685.837794:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175685.837798:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175685.837803:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.837808:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175685.837811:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175685.837834:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.837838:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175685.837847:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786175685.837853:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175685.837859:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175685.837863:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175685.837868:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175685.837873:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175685.837878:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175685.837884:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786175685.837892:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175685.837905:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235398912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175685.837925:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175685.837931:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175685.837959:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235398912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175685.837982:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175685.838009:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10370) 00000800:00000200:2.1:1786175685.838227:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175685.838230:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175685.838527:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175685.838529:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175685.838795:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175685.838798:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786175685.839121:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175685.839124:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1786175686.348535:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175686.348548:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175686.348551:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175686.348560:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175686.348566:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175686.348571:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175686.348575:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175686.348579:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175686.348596:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175687.372444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175687.372451:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175687.372453:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175687.372459:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175687.372465:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175687.372469:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175687.372472:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175687.372474:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175687.372477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175690.958581:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175690.958585:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175690.959338:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175690.959341:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175690.960760:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175690.960773:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175690.961565:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175690.961568:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175690.961944:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175690.961947:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00080000:3.0:1786175696.076579:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175696.076593:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175696.076596:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175696.076601:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175696.076607:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175696.076616:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10370 00000400:00000200:3.0:1786175696.076628:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175696.076636:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175696.076640:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175696.076644:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.076649:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175696.076652:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.076656:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175696.076661:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175696.076664:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175696.076668:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175696.076672:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175696.076675:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175696.076679:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175696.076683:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.076685:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175696.076688:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.076690:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175696.076694:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175696.076698:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175696.076701:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175696.076705:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175696.076709:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175696.076713:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.076716:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175696.076719:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175696.076731:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786175696.076737:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175696.076742:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175696.076744:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.076748:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175696.076752:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175696.076758:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175696.076767:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786175696.076775:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175696.076788:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235399680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175696.076802:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175696.076809:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175696.076815:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235399680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175696.076873:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175696.076880:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175696.076885:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175696.076888:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175696.076893:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175696.076898:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175696.076902:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10370 00000400:00000200:3.0:1786175696.076908:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175696.076912:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175696.076915:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175696.076919:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.076922:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175696.076929:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.076933:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175696.076942:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175696.076949:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175696.076961:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175696.076968:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175696.076978:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175696.076989:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175696.077013:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.077024:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175696.077042:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.077062:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175696.077080:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175696.077098:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175696.077110:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175696.077115:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175696.077136:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175696.077148:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.077151:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175696.077158:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175696.077174:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.077190:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175696.077200:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786175696.077207:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175696.077218:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175696.077220:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.077224:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175696.077230:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175696.077236:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175696.077261:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786175696.077270:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175696.077290:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235399808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175696.077319:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175696.077334:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175696.077420:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235399808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175696.077470:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175696.077481:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175696.077489:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175696.077491:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175696.077495:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175696.077504:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175696.077514:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10370 00000400:00000200:3.0:1786175696.077526:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175696.077533:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175696.077536:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175696.077540:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.077543:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175696.077549:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.077554:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175696.077566:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175696.077570:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175696.077573:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175696.077578:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175696.077593:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175696.077604:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175696.077620:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.077631:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175696.077643:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.077652:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175696.077669:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175696.077691:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175696.077697:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175696.077713:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175696.077727:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175696.077733:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.077744:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175696.077757:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175696.077765:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.077778:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175696.077787:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786175696.077799:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175696.077806:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175696.077813:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.077822:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1786175696.077824:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786175696.077828:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175696.077867:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175696.077870:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 02000000:00000010:3.0:1786175696.077878:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786175696.077891:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175696.077907:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235399936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175696.077953:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175696.077958:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175696.077979:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235399936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175696.078080:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175696.078093:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175696.078102:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175696.078112:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175696.078122:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175696.078140:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175696.078159:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10370 00000400:00000200:3.0:1786175696.078171:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175696.078187:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175696.078195:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175696.078208:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.078214:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175696.078222:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.078227:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175696.078231:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175696.078234:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175696.078239:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175696.078243:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175696.078249:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175696.078253:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175696.078259:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.078272:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175696.078276:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.078281:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175696.078285:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175696.078293:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175696.078299:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175696.078309:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175696.078320:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175696.078336:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.078351:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175696.078400:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175696.078418:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.078436:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175696.078450:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786175696.078462:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175696.078470:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175696.078483:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.078488:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175696.078494:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175696.078501:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175696.078509:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786175696.078516:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175696.078531:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235400064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175696.078620:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175696.078629:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175696.078658:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235400064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175696.078709:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175696.078733:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175696.078746:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175696.078759:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175696.078766:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175696.078786:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175696.078801:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10370 00000400:00000200:3.0:1786175696.078831:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175696.078872:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175696.078891:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175696.078910:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.078926:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175696.078945:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.078960:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175696.078977:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175696.078995:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175696.079007:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175696.079020:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175696.079036:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175696.079048:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175696.079076:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.079102:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175696.079119:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175696.079129:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175696.079144:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175696.079165:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175696.079173:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175696.079184:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175696.079192:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175696.079204:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.079210:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175696.079215:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175696.079221:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.079229:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175696.079236:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786175696.079243:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175696.079255:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175696.079259:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175696.079267:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175696.079271:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175696.079275:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175696.079287:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786175696.079294:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175696.079309:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235400192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175696.079347:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175696.079428:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175696.079457:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235400192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175696.079496:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175696.079514:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10380) 00000800:00000200:1.1:1786175696.079693:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175696.079697:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175696.080128:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175696.080132:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175696.080646:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175696.080650:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175696.083264:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175696.083269:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786175701.197099:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175701.197104:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175701.197663:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175701.197665:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175701.197872:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175701.197874:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175701.198055:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175701.198057:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175701.198241:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175701.198243:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175706.317686:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175706.318546:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175706.318892:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175706.319155:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175706.319426:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000100:3.0:1786175709.388809:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:3.0:1786175709.388900:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1786175709.388923:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175709.388932:0:2380:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:3.0:1786175709.388944:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175709.388952:0:2380:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1786175709.388962:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175709.388971:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:3.0:1786175709.388983:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175709.388994:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175709.389007:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28440 00000400:00000010:3.0:1786175709.389011:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28440. 00000100:00000001:3.0:1786175709.389019:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175709.389034:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328b9f80 x1872930235400960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786175731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175709.389076:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175709.389086:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235400960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786175731 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175709.389103:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175709.389107:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175709.389129:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175709.389136:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6dd00. 00000400:00000200:3.0:1786175709.389145:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175709.389154:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175709.389162:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28908 00000400:00000010:3.0:1786175709.389166:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28908. 00000100:00000001:3.0:1786175709.389173:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175709.389185:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328b9180 x1872930235401088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175709.389221:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175709.389238:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9180 x1872930235401088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175731 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175709.389258:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175709.389269:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175709.389293:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175709.389319:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d400. 00000400:00000200:3.0:1786175709.389333:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175709.389355:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175709.389484:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28c38 00000400:00000010:3.0:1786175709.389491:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28c38. 00000100:00000001:3.0:1786175709.389501:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175709.389512:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328bb100 x1872930235401216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175709.389535:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175709.389550:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb100 x1872930235401216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175731 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175709.389571:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175709.389588:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175709.389611:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175709.389622:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d700. 00000400:00000200:3.0:1786175709.389637:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175709.389667:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175709.389680:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c283b8 00000400:00000010:3.0:1786175709.389691:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c283b8. 00000100:00000001:3.0:1786175709.389706:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175709.389758:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328b8a80 x1872930235401344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175709.389808:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175709.389826:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8a80 x1872930235401344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175731 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175709.390019:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175709.390030:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175709.390058:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175709.390079:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d200. 00000400:00000200:3.0:1786175709.390094:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175709.390115:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175709.390131:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28550 00000400:00000010:3.0:1786175709.390141:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28550. 00000100:00000001:3.0:1786175709.390160:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175709.390177:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328bb800 x1872930235401472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175709.390202:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175709.390210:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb800 x1872930235401472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175731 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175709.390225:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175709.390233:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:2.1:1786175711.437588:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175711.437601:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786175711.439459:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175711.439468:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786175711.440097:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175711.440100:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786175711.440517:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175711.440520:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1786175711.441095:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175711.441108:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000100:00080000:3.0:1786175716.556488:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175716.556498:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175716.556501:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175716.556506:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175716.556510:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175716.556528:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10391 00000400:00000200:3.0:1786175716.556539:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175716.556546:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175716.556550:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175716.556553:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.556557:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175716.556560:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.556563:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175716.556567:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175716.556570:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175716.556573:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175716.556576:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175716.556579:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175716.556581:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175716.556585:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.556588:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175716.556590:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.556592:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175716.556595:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175716.556599:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175716.556602:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175716.556620:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175716.556624:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175716.556627:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.556629:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175716.556631:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175716.556669:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786175716.556674:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175716.556677:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175716.556679:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.556682:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175716.556685:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175716.556689:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175716.556697:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786175716.556704:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175716.556713:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235402240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175716.556724:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175716.556729:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175716.556734:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235402240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175716.556753:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175716.556756:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175716.556759:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175716.556761:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175716.556762:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175716.556765:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175716.556768:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10391 00000400:00000200:3.0:1786175716.556773:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175716.556778:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175716.556782:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175716.556787:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.556790:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175716.556794:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.556797:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175716.556801:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175716.556806:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175716.556812:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175716.556817:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175716.556822:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175716.556830:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175716.556843:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.556850:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175716.556855:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.556859:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175716.556864:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175716.556872:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175716.556887:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175716.556894:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175716.556903:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175716.556912:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.556918:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175716.556926:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175716.556933:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.556937:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175716.556947:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786175716.556953:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175716.556959:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175716.556963:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.556968:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175716.556972:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175716.556978:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175716.556986:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786175716.556993:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175716.557003:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235402368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175716.557019:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175716.557023:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175716.557032:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235402368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175716.557047:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175716.557053:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175716.557063:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175716.557068:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175716.557071:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175716.557079:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175716.557084:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10391 00000400:00000200:3.0:1786175716.557093:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175716.557105:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175716.557114:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175716.557123:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557126:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175716.557129:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557134:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175716.557140:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175716.557145:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175716.557151:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175716.557156:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175716.557163:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000200:2.1:1786175716.557164:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:3.0:1786175716.557170:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000200:2.1:1786175716.557171:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000100:00000040:3.0:1786175716.557177:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557185:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175716.557192:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557198:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175716.557200:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175716.557206:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557212:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175716.557218:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175716.557223:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175716.557229:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.557235:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175716.557238:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175716.557242:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.557248:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175716.557256:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786175716.557262:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175716.557267:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175716.557270:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.557275:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175716.557278:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175716.557282:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175716.557289:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786175716.557296:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175716.557306:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235402496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175716.557329:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175716.557335:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175716.557365:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235402496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175716.557463:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175716.557472:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175716.557487:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175716.557493:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175716.557500:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175716.557507:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175716.557517:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10391 00000400:00000200:3.0:1786175716.557531:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175716.557542:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175716.557551:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175716.557561:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557570:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175716.557578:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557583:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175716.557605:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175716.557609:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175716.557613:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175716.557617:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175716.557620:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175716.557627:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175716.557636:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557655:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175716.557660:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557663:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175716.557666:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175716.557671:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557678:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175716.557683:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175716.557688:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175716.557696:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.557698:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175716.557704:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175716.557708:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.557712:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175716.557719:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786175716.557727:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175716.557734:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175716.557739:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.557744:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175716.557748:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175716.557752:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175716.557758:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786175716.557765:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175716.557778:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235402624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175716.557814:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175716.557822:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175716.557835:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235402624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175716.557856:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175716.557864:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175716.557872:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175716.557877:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175716.557881:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175716.557885:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175716.557893:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10391 00000400:00000200:3.0:1786175716.557908:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175716.557921:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175716.557928:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175716.557939:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557946:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175716.557953:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.557960:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175716.557968:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175716.557973:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175716.557976:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175716.557981:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175716.557986:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175716.557994:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175716.558001:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.558006:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175716.558012:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175716.558016:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175716.558022:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175716.558028:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175716.558032:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175716.558035:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175716.558040:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175716.558047:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.558052:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175716.558056:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175716.558061:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.558065:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175716.558075:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786175716.558082:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175716.558087:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175716.558091:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175716.558097:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175716.558103:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175716.558110:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175716.558121:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786175716.558133:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175716.558148:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235402752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175716.558185:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1786175716.558186:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786175716.558190:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175716.558192:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000100:00000040:3.0:1786175716.558199:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235402752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175716.558218:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175716.558227:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10401) 00000800:00000200:2.1:1786175716.558583:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175716.558586:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786175716.558906:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175716.558908:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786175716.559231:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175716.559233:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000100:00000001:3.0:1786175721.548657:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175721.548671:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175721.548676:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175721.548688:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175721.548699:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175721.548719:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175721.548724:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175721.548729:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175721.548735:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175721.676507:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175721.676528:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175721.676532:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175721.676540:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175721.676549:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175721.676563:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10396 00000400:00000200:3.0:1786175721.676580:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175721.676624:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175721.676629:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175721.676634:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.676639:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175721.676642:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.676646:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175721.676651:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175721.676654:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175721.676658:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175721.676663:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175721.676666:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175721.676669:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175721.676672:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.676674:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175721.676676:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.676678:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175721.676682:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175721.676685:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175721.676688:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175721.676692:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175721.676697:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175721.676700:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.676703:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175721.676706:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175721.676721:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786175721.676725:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175721.676731:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175721.676732:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.676737:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175721.676741:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175721.676747:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175721.676760:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786175721.676769:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175721.676782:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235402880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175721.676795:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175721.676801:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175721.676807:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235402880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175721.676844:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175721.676848:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175721.676854:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175721.676855:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175721.676857:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175721.676861:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175721.676864:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10396 00000400:00000200:3.0:1786175721.676868:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175721.676871:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175721.676873:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175721.676876:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.676878:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175721.676880:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.676883:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175721.676887:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175721.676890:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175721.676894:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175721.676897:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175721.676902:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175721.676907:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175721.676914:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.676918:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175721.676922:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.676929:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175721.676944:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175721.676959:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175721.676970:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175721.676986:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175721.677004:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175721.677011:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.677015:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175721.677024:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175721.677042:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.677054:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175721.677075:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786175721.677085:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175721.677098:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175721.677103:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.677111:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175721.677115:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175721.677123:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175721.677134:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786175721.677142:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175721.677161:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235403008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175721.677184:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175721.677193:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175721.677205:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235403008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175721.677243:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175721.677260:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175721.677275:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175721.677287:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175721.677296:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175721.677311:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175721.677319:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10396 00000400:00000200:3.0:1786175721.677333:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175721.677348:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175721.677421:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175721.677428:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.677435:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175721.677439:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.677450:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175721.677461:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175721.677465:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175721.677468:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175721.677474:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175721.677482:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175721.677487:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175721.677503:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.677513:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175721.677516:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.677519:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175721.677527:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175721.677532:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175721.677540:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175721.677550:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175721.677562:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175721.677573:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.677582:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175721.677587:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175721.677619:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.677629:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175721.677642:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786175721.677648:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175721.677653:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175721.677659:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.677668:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175721.677675:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175721.677682:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175721.677693:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786175721.677703:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175721.677715:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235403136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175721.677750:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175721.677755:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175721.677775:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235403136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175721.677811:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175721.677813:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786175721.677829:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.1:1786175721.677830:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000100:00080000:3.0:1786175721.677851:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175721.677860:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175721.677874:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175721.677890:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175721.677905:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10396 00000400:00000200:3.0:1786175721.677931:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175721.677946:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175721.677961:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175721.677982:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.677996:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175721.678004:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.678021:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175721.678034:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175721.678038:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175721.678046:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175721.678052:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175721.678057:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175721.678069:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175721.678081:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.678085:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175721.678089:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.678092:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175721.678103:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175721.678111:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175721.678124:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175721.678135:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175721.678143:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175721.678157:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.678161:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175721.678166:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175721.678178:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.678183:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175721.678189:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786175721.678201:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175721.678210:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175721.678213:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.678226:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175721.678231:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175721.678238:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175721.678253:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786175721.678268:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175721.678281:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235403264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175721.678296:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175721.678302:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175721.678318:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235403264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175721.678413:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175721.678423:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175721.678438:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175721.678446:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175721.678449:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175721.678455:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175721.678472:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10396 00000400:00000200:3.0:1786175721.678495:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175721.678512:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175721.678528:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175721.678544:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.678553:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175721.678567:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.678581:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175721.678611:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175721.678613:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175721.678621:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175721.678628:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175721.678640:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175721.678649:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175721.678659:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.678667:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175721.678676:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175721.678682:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175721.678690:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175721.678703:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175721.678708:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175721.678714:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175721.678724:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175721.678741:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.678745:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175721.678750:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175721.678758:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.678762:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175721.678785:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786175721.678798:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175721.678811:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175721.678824:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175721.678830:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175721.678834:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175721.678843:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175721.678854:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786175721.678866:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175721.678889:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235403392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175721.678939:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175721.678958:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175721.678981:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235403392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175721.679013:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175721.679022:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10406) 00000800:00000200:2.1:1786175721.679263:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175721.679266:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786175721.679797:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175721.679801:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786175721.680144:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175721.680147:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786175721.680499:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175721.680502:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1786175722.572432:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175722.572439:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175722.572441:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175722.572446:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175722.572454:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175722.572457:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175722.572468:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175722.572470:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175722.572473:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175723.596561:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175723.596571:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175723.596599:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175723.596608:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175723.596634:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175723.596644:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175723.596655:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175723.596663:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175723.596672:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175724.620452:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175724.620459:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175724.620461:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175724.620465:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175724.620472:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175724.620479:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175724.620494:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175724.620498:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175724.620508:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175725.644452:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175725.644461:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175725.644464:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175725.644471:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175725.644476:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175725.644481:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175725.644485:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175725.644488:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175725.644492:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175726.668498:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175726.668510:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175726.668516:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175726.668524:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175726.668535:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175726.668567:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175726.668575:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175726.668596:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175726.668600:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175726.797307:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175726.797315:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786175726.799905:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175726.799909:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786175726.800284:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175726.800287:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786175726.800763:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175726.800767:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786175726.801124:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175726.801127:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1786175727.692470:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175727.692480:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175727.692484:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175727.692490:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175727.692496:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175727.692502:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175727.692506:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175727.692509:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175727.692514:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175728.716437:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175728.716446:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175728.716449:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175728.716455:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175728.716459:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175728.716463:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175728.716467:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175728.716482:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175728.716485:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175729.740496:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175729.740506:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175729.740527:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175729.740536:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175729.740545:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175729.740570:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175729.740577:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175729.740582:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175729.740587:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175729.740656:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175729.740659:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175729.740663:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175730.764433:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175730.764443:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175730.764447:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175730.764455:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175730.764461:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175730.764466:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175730.764473:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175730.764475:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175730.764478:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175731.788482:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175731.788512:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175731.788516:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175731.788524:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175731.788548:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175731.788556:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175731.788562:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175731.788566:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175731.788576:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175731.788639:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175731.788644:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175731.788648:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786175731.917520:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175731.917557:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175731.918773:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175731.918776:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175731.919230:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175731.919233:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175731.919685:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175731.919688:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175731.920234:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175731.920237:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1786175732.812526:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175732.812537:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175732.812541:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175732.812549:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175732.812555:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175732.812560:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175732.812565:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175732.812569:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175732.812576:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175733.836542:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175733.836552:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175733.836554:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175733.836560:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175733.836565:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175733.836572:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175733.836581:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175733.836584:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175733.836589:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175733.836639:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175733.836642:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175733.836646:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175734.860527:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175734.860538:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175734.860542:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175734.860550:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175734.860560:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175734.860568:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175734.860580:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175734.860586:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175734.860597:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175735.884520:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175735.884530:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175735.884533:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175735.884541:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175735.884547:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175735.884553:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175735.884558:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175735.884562:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175735.884567:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175736.908518:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175736.908530:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175736.908533:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175736.908541:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175736.908547:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175736.908552:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175736.908556:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175736.908564:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175736.908574:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175737.036583:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175737.036602:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175737.036607:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175737.036613:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175737.036620:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175737.036633:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10411 00000400:00000200:3.0:1786175737.036646:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175737.036654:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175737.036659:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175737.036663:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.036668:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175737.036671:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.036674:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175737.036680:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175737.036683:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175737.036686:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175737.036690:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175737.036693:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175737.036697:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175737.036700:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.036703:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175737.036706:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.036708:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175737.036712:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175737.036717:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175737.036720:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175737.036724:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175737.036740:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175737.036743:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.036746:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175737.036749:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175737.036761:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786175737.036766:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175737.036772:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175737.036775:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.036778:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175737.036783:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175737.036788:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175737.036798:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786175737.036806:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175737.036818:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235404800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175737.036832:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175737.036840:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175737.036846:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235404800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175737.036879:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175737.036883:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175737.036888:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175737.036890:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175737.036892:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175737.036896:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175737.036899:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10411 00000400:00000200:3.0:1786175737.036904:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175737.036908:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175737.036911:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175737.036915:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.036918:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175737.036921:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.036925:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175737.036928:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175737.036930:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175737.036933:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175737.036936:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175737.036938:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175737.036941:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175737.036945:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.036948:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175737.036951:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.036954:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175737.036961:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175737.036967:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175737.036976:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175737.036980:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175737.036991:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175737.036996:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.037000:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175737.037006:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175737.037030:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.037036:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175737.037044:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786175737.037051:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175737.037059:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175737.037064:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.037070:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175737.037075:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175737.037082:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175737.037091:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786175737.037099:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175737.037112:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235404928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175737.037130:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175737.037136:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175737.037147:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235404928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175737.037174:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175737.037182:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175737.037193:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175737.037195:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175737.037199:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175737.037207:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175737.037215:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10411 00000400:00000200:3.0:1786175737.037226:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175737.037236:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175737.037239:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175737.037248:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.037256:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175737.037261:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.037269:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175737.037276:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175737.037279:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175737.037284:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175737.037289:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175737.037292:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175737.037299:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175737.037309:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.037317:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175737.037321:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.037331:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175737.037335:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175737.037344:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175737.037349:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175737.037354:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175737.037399:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175737.037403:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.037406:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175737.037408:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175737.037412:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.037417:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175737.037425:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786175737.037428:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175737.037434:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175737.037439:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.037468:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175737.037472:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175737.037479:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175737.037490:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786175737.037501:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175737.037521:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235405056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175737.037558:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175737.037568:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175737.037584:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235405056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175737.037625:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175737.037631:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175737.037639:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175737.037646:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175737.037651:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175737.037657:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175737.037667:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10411 00000400:00000200:3.0:1786175737.037680:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175737.037692:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000200:1.1:1786175737.037693:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786175737.037698:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1786175737.037702:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000040:3.0:1786175737.037708:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.037715:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175737.037723:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.037729:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175737.037735:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175737.037738:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175737.037742:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175737.037748:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175737.037756:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175737.037764:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175737.037772:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.037778:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175737.037785:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.037794:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175737.037799:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175737.037808:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175737.037817:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175737.037825:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175737.037832:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175737.037838:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.037844:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175737.037850:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175737.037856:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.037864:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175737.037875:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786175737.037882:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175737.037888:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175737.037893:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.037897:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175737.037902:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175737.037907:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175737.037915:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786175737.037923:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175737.037936:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235405184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175737.037949:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175737.037956:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175737.037963:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235405184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175737.037982:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175737.037986:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175737.037996:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175737.037999:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175737.038002:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175737.038011:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175737.038017:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10411 00000400:00000200:3.0:1786175737.038026:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175737.038034:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175737.038043:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175737.038052:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.038057:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175737.038065:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.038073:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175737.038080:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175737.038085:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175737.038090:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175737.038095:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175737.038100:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175737.038104:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175737.038118:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.038124:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175737.038133:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175737.038137:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175737.038145:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175737.038158:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175737.038167:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175737.038173:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175737.038180:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175737.038189:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.038197:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175737.038203:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175737.038209:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.038216:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175737.038226:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786175737.038234:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175737.038239:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175737.038245:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175737.038249:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175737.038253:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175737.038257:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175737.038264:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786175737.038271:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175737.038287:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235405312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175737.038304:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175737.038312:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175737.038325:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235405312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175737.038346:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175737.038390:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10421) 00000800:00000200:1.1:1786175737.038768:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175737.038771:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175737.039200:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175737.039203:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175737.039963:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175737.039967:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175737.040312:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175737.040316:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1786175737.932503:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175737.932513:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175737.932517:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175737.932525:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175737.932532:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175737.932538:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175737.932543:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175737.932547:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175737.932554:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175738.956606:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175738.956617:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175738.956620:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175738.956627:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175738.956632:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175738.956638:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175738.956642:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175738.956646:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175738.956651:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175739.980474:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175739.980481:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175739.980484:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175739.980489:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175739.980494:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175739.980499:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175739.980502:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175739.980505:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175739.980509:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175741.004496:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175741.004504:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175741.004507:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175741.004513:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175741.004518:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175741.004523:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175741.004527:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175741.004531:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175741.004535:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175742.028503:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175742.028516:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175742.028523:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175742.028537:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175742.028550:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175742.028556:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175742.028561:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175742.028566:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175742.028572:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175742.157343:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175742.157352:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175742.158239:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175742.158242:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175742.158564:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175742.158567:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175742.158857:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175742.158859:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175742.159131:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175742.159134:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1786175743.052449:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175743.052458:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175743.052462:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175743.052470:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175743.052476:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175743.052480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175743.052483:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175743.052485:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175743.052489:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175744.076522:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175744.076534:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175744.076536:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175744.076545:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175744.076555:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175744.076562:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175744.076567:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175744.076572:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175744.076580:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175745.100565:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175745.100579:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175745.100583:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175745.100594:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175745.100604:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175745.100611:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175745.100617:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175745.100623:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175745.100631:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175746.124415:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175746.124421:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175746.124423:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175746.124428:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175746.124431:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175746.124434:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175746.124437:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175746.124440:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175746.124445:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.148449:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175747.148458:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175747.148461:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175747.148467:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175747.148473:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175747.148477:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175747.148483:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175747.148486:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175747.148489:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175747.276484:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175747.276495:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175747.276498:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175747.276501:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175747.276505:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175747.276512:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10421 00000400:00000200:3.0:1786175747.276521:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175747.276527:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175747.276530:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175747.276533:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276537:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175747.276539:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276542:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175747.276546:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175747.276549:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175747.276554:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175747.276558:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175747.276562:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175747.276565:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175747.276569:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276573:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175747.276575:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276578:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175747.276582:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175747.276586:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276590:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175747.276594:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175747.276598:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175747.276602:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.276628:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175747.276631:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175747.276640:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786175747.276644:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175747.276648:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175747.276650:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.276653:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175747.276656:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175747.276660:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175747.276667:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786175747.276685:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175747.276695:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235406080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175747.276705:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175747.276710:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175747.276715:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235406080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175747.276736:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175747.276740:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175747.276743:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175747.276744:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175747.276746:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175747.276748:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175747.276750:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10421 00000400:00000200:3.0:1786175747.276755:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175747.276758:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175747.276760:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175747.276763:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276765:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175747.276767:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276769:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175747.276771:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175747.276772:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175747.276773:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175747.276775:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175747.276777:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175747.276779:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175747.276781:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276783:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175747.276784:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276786:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175747.276788:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175747.276797:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276799:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175747.276801:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175747.276806:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175747.276808:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.276810:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175747.276815:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175747.276818:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.276820:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175747.276824:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786175747.276827:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175747.276830:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175747.276831:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.276833:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175747.276834:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175747.276835:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175747.276839:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786175747.276842:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175747.276850:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235406208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175747.276858:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175747.276860:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175747.276865:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235406208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175747.276873:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175747.276875:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175747.276878:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175747.276879:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175747.276882:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175747.276886:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175747.276894:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10421 00000400:00000200:3.0:1786175747.276905:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175747.276922:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175747.276927:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175747.276936:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276949:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175747.276960:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.276973:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175747.276984:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175747.276997:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175747.277015:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175747.277019:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175747.277024:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175747.277041:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175747.277062:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277074:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175747.277082:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277086:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175747.277094:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175747.277101:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277105:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175747.277107:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175747.277112:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175747.277125:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.277142:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175747.277147:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175747.277159:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.277164:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175747.277169:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786175747.277180:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175747.277187:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175747.277198:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.277206:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175747.277212:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175747.277221:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175747.277236:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786175747.277246:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175747.277263:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235406336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175747.277300:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175747.277308:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175747.277316:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235406336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175747.277340:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175747.277388:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175747.277393:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175747.277394:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175747.277396:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175747.277399:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175747.277403:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10421 00000400:00000200:3.0:1786175747.277408:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175747.277414:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175747.277416:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175747.277420:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277423:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175747.277426:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277429:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175747.277432:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175747.277433:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175747.277435:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175747.277439:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175747.277440:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175747.277443:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175747.277446:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277449:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175747.277451:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277453:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175747.277456:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175747.277459:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277462:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175747.277464:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175747.277468:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175747.277475:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.277480:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175747.277490:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175747.277494:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.277496:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175747.277501:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786175747.277506:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175747.277511:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175747.277521:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.277525:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175747.277527:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175747.277530:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175747.277537:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786175747.277546:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175747.277563:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235406464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175747.277623:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175747.277632:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175747.277642:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235406464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175747.277692:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175747.277699:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175747.277704:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175747.277708:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175747.277731:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175747.277738:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175747.277752:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10421 00000400:00000200:3.0:1786175747.277775:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:2.1:1786175747.277796:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:3.0:1786175747.277803:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000200:2.1:1786175747.277806:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00080000:3.0:1786175747.277808:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175747.277829:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277839:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175747.277847:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277854:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175747.277866:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175747.277895:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175747.277901:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175747.277907:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175747.277915:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175747.277921:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175747.277934:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277967:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175747.277975:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175747.277982:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175747.277988:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175747.278023:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175747.278041:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175747.278050:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175747.278056:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175747.278070:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.278093:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175747.278100:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175747.278107:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.278121:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175747.278128:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786175747.278162:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175747.278169:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175747.278180:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175747.278186:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175747.278193:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175747.278205:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175747.278218:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786175747.278232:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175747.278250:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235406592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175747.278263:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175747.278272:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175747.278280:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235406592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175747.278299:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175747.278309:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10431) 00000800:00000200:2.1:1786175747.278771:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175747.278774:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175747.279312:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175747.279318:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175747.279810:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175747.279812:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175747.280141:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175747.280143:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1786175748.172466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175748.172476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175748.172480:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175748.172486:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175748.172491:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175748.172496:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175748.172500:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175748.172504:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175748.172508:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175749.196421:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175749.196431:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175749.196435:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175749.196442:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175749.196450:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175749.196455:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175749.196460:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175749.196463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175749.196467:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175750.220424:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175750.220433:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175750.220436:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175750.220442:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175750.220454:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175750.220458:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175750.220461:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175750.220465:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175750.220470:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175751.244466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175751.244473:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175751.244477:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175751.244484:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175751.244488:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175751.244495:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175751.244501:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175751.244504:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175751.244509:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175752.268417:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175752.268424:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175752.268426:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175752.268430:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175752.268436:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175752.268439:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175752.268441:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175752.268443:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175752.268447:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175752.397040:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175752.397047:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175752.397709:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175752.397712:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175752.397962:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175752.397964:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175752.398183:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175752.398185:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175752.398514:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175752.398516:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1786175753.292547:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175753.292556:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175753.292560:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175753.292568:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175753.292574:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175753.292578:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175753.292582:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175753.292585:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175753.292589:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175754.316429:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175754.316436:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175754.316439:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175754.316447:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175754.316452:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175754.316457:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175754.316464:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175754.316466:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175754.316469:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175755.340451:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175755.340458:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175755.340462:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175755.340466:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175755.340470:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175755.340475:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175755.340478:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175755.340481:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175755.340485:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175756.364407:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175756.364414:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175756.364416:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175756.364421:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175756.364425:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175756.364428:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175756.364431:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175756.364433:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175756.364435:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.388528:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175757.388538:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175757.388543:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175757.388552:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175757.388558:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175757.388563:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175757.388567:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175757.388570:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175757.388574:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175757.516534:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175757.516561:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175757.516565:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175757.516571:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175757.516578:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175757.516589:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10431 00000400:00000200:3.0:1786175757.516602:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175757.516612:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175757.516617:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175757.516621:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.516626:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175757.516630:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.516633:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175757.516639:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175757.516642:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175757.516646:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175757.516651:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175757.516654:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175757.516657:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175757.516660:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.516663:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175757.516666:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.516668:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175757.516673:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175757.516678:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175757.516681:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175757.516685:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175757.516690:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175757.516694:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.516697:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175757.516701:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175757.516714:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786175757.516719:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175757.516725:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175757.516727:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.516732:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175757.516736:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175757.516743:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175757.516754:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786175757.516815:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175757.516830:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235407360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175757.516845:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175757.516851:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175757.516858:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235407360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175757.516889:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175757.516894:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175757.516899:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175757.516901:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175757.516904:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175757.516908:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175757.516913:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10431 00000400:00000200:3.0:1786175757.516921:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175757.516927:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175757.516931:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175757.516937:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.516942:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175757.516947:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.516951:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175757.516956:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175757.516960:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175757.516964:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175757.516968:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175757.516973:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175757.516978:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175757.516986:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.516992:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175757.517007:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517016:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175757.517023:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175757.517033:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517038:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175757.517045:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175757.517056:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175757.517065:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.517072:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175757.517079:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175757.517090:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.517097:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175757.517107:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786175757.517114:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175757.517121:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175757.517127:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.517131:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175757.517136:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175757.517142:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175757.517151:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786175757.517163:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175757.517187:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235407488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175757.517204:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175757.517211:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175757.517219:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235407488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175757.517240:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175757.517275:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175757.517284:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175757.517300:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175757.517307:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175757.517317:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175757.517326:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10431 00000400:00000200:3.0:1786175757.517340:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175757.517350:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175757.517399:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175757.517405:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517408:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175757.517412:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517415:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175757.517419:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175757.517421:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175757.517423:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175757.517427:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175757.517435:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175757.517443:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175757.517450:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517454:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175757.517462:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517468:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175757.517472:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175757.517482:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517489:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175757.517493:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175757.517500:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175757.517506:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.517512:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175757.517517:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175757.517526:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.517533:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175757.517544:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786175757.517565:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175757.517570:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175757.517577:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.517585:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175757.517590:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175757.517597:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175757.517608:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786175757.517616:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175757.517630:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235407616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175757.517649:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175757.517655:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175757.517659:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786175757.517667:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235407616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1786175757.517668:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000100:00000001:3.0:1786175757.517698:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175757.517704:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175757.517715:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175757.517723:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175757.517728:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175757.517734:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175757.517742:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10431 00000400:00000200:3.0:1786175757.517756:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175757.517769:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175757.517772:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175757.517783:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517791:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175757.517798:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517803:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175757.517808:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175757.517814:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175757.517818:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175757.517824:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175757.517832:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175757.517840:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175757.517850:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517856:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175757.517864:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517868:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175757.517871:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175757.517876:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175757.517884:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175757.517888:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175757.517894:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175757.517901:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.517905:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175757.517912:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175757.517919:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.517925:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175757.517936:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786175757.517943:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175757.517952:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175757.517955:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.517960:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175757.517967:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175757.517973:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175757.517996:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786175757.518009:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175757.518030:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235407744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175757.518066:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175757.518074:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175757.518085:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235407744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175757.518113:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175757.518123:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175757.518140:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175757.518145:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175757.518151:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175757.518161:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175757.518171:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10431 00000400:00000200:3.0:1786175757.518186:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175757.518195:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175757.518202:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175757.518214:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.518222:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175757.518230:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.518239:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175757.518267:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175757.518272:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175757.518275:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175757.518289:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175757.518295:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175757.518303:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175757.518318:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.518324:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175757.518334:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175757.518344:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175757.518353:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175757.518394:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175757.518398:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175757.518403:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175757.518413:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175757.518421:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.518428:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175757.518435:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175757.518447:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.518453:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175757.518463:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786175757.518470:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175757.518479:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175757.518486:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175757.518494:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175757.518498:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175757.518503:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175757.518517:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786175757.518529:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175757.518543:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235407872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175757.518577:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175757.518583:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175757.518597:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235407872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175757.518623:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175757.518630:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10442) 00000800:00000200:1.1:1786175757.518848:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175757.518854:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786175757.519308:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175757.519321:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786175757.519718:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175757.519722:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786175757.520140:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175757.520144:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000100:00000001:3.0:1786175758.412462:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175758.412485:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175758.412491:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175758.412502:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175758.412509:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175758.412517:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175758.412525:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175758.412532:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175758.412539:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175759.436517:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175759.436533:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175759.436542:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175759.436555:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175759.436571:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175759.436581:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175759.436590:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175759.436595:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175759.436602:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175760.460430:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175760.460438:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175760.460443:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175760.460450:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175760.460455:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175760.460459:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175760.460462:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175760.460465:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175760.460469:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175761.484426:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175761.484433:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175761.484438:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175761.484442:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175761.484448:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175761.484452:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175761.484455:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175761.484458:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175761.484460:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175762.508480:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175762.508491:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175762.508496:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175762.508503:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175762.508511:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175762.508517:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175762.508526:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175762.508531:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175762.508535:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175762.637559:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175762.637570:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786175762.638883:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175762.638886:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786175762.639342:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175762.639346:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786175762.639710:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175762.639713:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786175762.640439:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175762.640443:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000100:00000001:3.0:1786175763.532453:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175763.532464:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175763.532472:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175763.532481:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175763.532491:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175763.532498:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175763.532502:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175763.532506:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175763.532510:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175764.556474:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175764.556486:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175764.556490:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175764.556498:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175764.556506:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175764.556512:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175764.556516:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175764.556519:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175764.556524:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175765.580430:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175765.580437:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175765.580439:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175765.580443:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175765.580447:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175765.580450:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175765.580452:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175765.580455:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175765.580458:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175766.604452:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175766.604462:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175766.604465:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175766.604473:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175766.604496:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175766.604500:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175766.604506:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175766.604513:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175766.604518:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.628454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175767.628465:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175767.628468:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175767.628476:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175767.628480:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175767.628486:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175767.628490:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175767.628494:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175767.628498:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175767.756496:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175767.756506:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175767.756509:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175767.756513:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175767.756518:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175767.756527:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10442 00000400:00000200:3.0:1786175767.756538:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175767.756545:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175767.756548:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175767.756551:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.756554:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175767.756557:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.756559:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175767.756565:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175767.756568:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175767.756572:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175767.756576:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175767.756580:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175767.756583:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175767.756588:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.756591:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175767.756593:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.756596:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175767.756601:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175767.756606:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175767.756610:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175767.756613:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175767.756618:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175767.756623:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.756626:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175767.756629:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175767.756640:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786175767.756656:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175767.756660:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175767.756662:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.756665:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175767.756668:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175767.756672:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175767.756680:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786175767.756687:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175767.756696:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235408640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175767.756707:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175767.756713:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175767.756718:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235408640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175767.756741:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175767.756744:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175767.756748:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175767.756749:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175767.756751:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175767.756753:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175767.756755:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10442 00000400:00000200:3.0:1786175767.756759:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175767.756763:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175767.756765:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175767.756768:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.756771:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175767.756775:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.756777:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175767.756780:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175767.756782:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175767.756784:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175767.756786:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175767.756788:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175767.756790:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175767.756797:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.756800:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175767.756805:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.756810:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175767.756814:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175767.756820:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175767.756823:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175767.756829:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175767.756860:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175767.756871:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.756876:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175767.756886:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175767.756895:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.756900:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175767.756911:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786175767.756919:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175767.756926:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175767.756930:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.756936:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175767.756939:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175767.756946:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175767.756955:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786175767.756963:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175767.756979:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235408768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175767.757004:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175767.757011:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175767.757023:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235408768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175767.757046:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175767.757051:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175767.757062:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175767.757068:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175767.757071:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175767.757078:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175767.757089:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10442 00000400:00000200:3.0:1786175767.757106:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175767.757118:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175767.757127:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175767.757136:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.757202:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175767.757213:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.757220:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175767.757229:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175767.757233:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175767.757239:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175767.757247:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175767.757256:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175767.757264:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175767.757275:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.757283:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175767.757290:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.757294:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175767.757302:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175767.757309:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175767.757314:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175767.757319:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175767.757326:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175767.757332:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.757336:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175767.757342:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175767.757350:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.757353:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175767.757369:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175767.757397:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000100:00000010:3.0:1786175767.757411:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786175767.757420:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175767.757430:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175767.757434:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.757442:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175767.757447:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175767.757455:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175767.757466:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786175767.757478:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175767.757496:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235408896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175767.757521:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175767.757525:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175767.757531:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235408896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175767.757554:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175767.757560:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175767.757565:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175767.757570:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175767.757573:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175767.757584:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175767.757594:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10442 00000400:00000200:3.0:1786175767.757610:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175767.757623:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175767.757632:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175767.757642:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.757658:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175767.757662:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.757666:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175767.757670:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175767.757672:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175767.757677:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175767.757680:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175767.757687:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175767.757692:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175767.757701:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.757706:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175767.757711:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.757716:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175767.757723:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175767.757732:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175767.757737:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175767.757741:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175767.757747:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175767.757753:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.757759:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175767.757763:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175767.757770:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.757775:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175767.757783:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786175767.757788:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175767.757792:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175767.757796:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.757798:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175767.757802:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175767.757805:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175767.757832:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786175767.757842:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175767.757857:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235409024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175767.757885:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175767.757893:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175767.757911:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235409024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175767.757936:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175767.757946:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175767.757959:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175767.757966:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175767.757971:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175767.757981:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175767.757990:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10442 00000400:00000200:3.0:1786175767.758004:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175767.758016:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175767.758026:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175767.758037:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.758044:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175767.758054:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.758063:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175767.758071:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175767.758076:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175767.758083:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175767.758089:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175767.758099:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175767.758110:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175767.758122:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.758128:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175767.758136:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175767.758142:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175767.758171:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175767.758213:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175767.758230:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175767.758237:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175767.758247:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175767.758252:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.758256:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175767.758263:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175767.758271:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.758275:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175767.758282:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786175767.758289:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175767.758296:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175767.758300:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175767.758305:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175767.758307:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175767.758312:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175767.758319:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786175767.758329:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175767.758341:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235409152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175767.758408:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175767.758413:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175767.758427:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235409152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175767.758458:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175767.758470:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10452) 00000800:00000200:2.1:1786175767.758540:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175767.758543:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175767.758914:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175767.758918:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175767.759318:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175767.759322:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175767.759751:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175767.759753:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000100:00000001:3.0:1786175768.652454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175768.652464:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175768.652467:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175768.652476:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175768.652482:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175768.652488:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175768.652494:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175768.652497:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175768.652500:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175769.676504:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175769.676516:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175769.676520:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175769.676528:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175769.676535:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175769.676545:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175769.676553:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175769.676562:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175769.676573:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175770.700482:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175770.700499:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175770.700509:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175770.700524:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175770.700538:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175770.700548:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175770.700557:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175770.700562:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175770.700569:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175771.724462:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175771.724473:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175771.724477:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175771.724484:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175771.724490:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175771.724497:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175771.724501:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175771.724507:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175771.724513:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175772.364584:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175772.364593:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175772.364596:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175772.364603:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175772.364610:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175772.364615:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175772.364620:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175772.364623:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175772.364627:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175772.748472:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175772.748485:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175772.748488:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175772.748496:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175772.748503:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175772.748508:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175772.748525:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175772.748530:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175772.748535:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175772.877499:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175772.878387:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175772.878790:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175772.879065:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175772.879315:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786175773.388490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175773.388513:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175773.388515:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175773.388522:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175773.388527:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175773.388533:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175773.388537:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175773.388540:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175773.388545:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175773.772487:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175773.772500:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175773.772502:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175773.772508:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175773.772514:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175773.772519:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175773.772522:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175773.772525:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175773.772529:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175774.412519:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175774.412534:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175774.412538:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175774.412548:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175774.412555:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175774.412561:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175774.412565:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175774.412571:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175774.412576:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175774.796454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175774.796464:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175774.796467:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175774.796474:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175774.796480:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175774.796484:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175774.796488:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175774.796491:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175774.796495:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175775.436484:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175775.436497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175775.436502:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175775.436511:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175775.436518:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175775.436524:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175775.436529:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175775.436533:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175775.436540:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175775.820445:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175775.820454:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175775.820457:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175775.820464:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175775.820470:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175775.820476:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175775.820481:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175775.820486:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175775.820493:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175776.460475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175776.460485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175776.460488:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175776.460494:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175776.460498:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175776.460506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175776.460510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175776.460514:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175776.460518:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175776.844454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175776.844462:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175776.844468:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175776.844475:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175776.844485:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175776.844493:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175776.844497:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175776.844501:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175776.844506:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175777.484501:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175777.484510:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175777.484513:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175777.484536:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175777.484543:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175777.484549:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175777.484554:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175777.484558:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175777.484562:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175777.868486:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175777.868494:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175777.868496:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175777.868503:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175777.868507:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175777.868512:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175777.868514:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175777.868517:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175777.868521:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175777.997440:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175777.997449:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175777.998613:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175777.998618:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175777.999110:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175777.999114:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175777.999580:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175777.999584:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175778.000000:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175778.000003:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786175778.508478:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175778.508490:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175778.508494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175778.508512:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175778.508518:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175778.508521:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175778.508524:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175778.508530:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175778.508535:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175778.892496:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175778.892507:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175778.892510:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175778.892518:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175778.892525:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175778.892533:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175778.892542:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175778.892546:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175778.892554:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175779.532451:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175779.532459:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175779.532461:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175779.532467:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175779.532470:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175779.532474:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175779.532476:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175779.532478:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175779.532481:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175779.916436:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175779.916444:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175779.916447:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175779.916455:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175779.916461:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175779.916468:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175779.916476:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175779.916483:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175779.916492:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175780.940464:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175780.940477:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175780.940483:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175780.940493:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175780.940502:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175780.940510:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175780.940517:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175780.940521:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175780.940526:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175781.580483:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175781.580491:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175781.580493:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175781.580498:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175781.580502:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175781.580505:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175781.580507:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175781.580509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175781.580512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175781.964437:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175781.964445:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175781.964447:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175781.964451:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175781.964455:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175781.964458:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175781.964471:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175781.964474:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175781.964477:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175782.988482:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175782.988493:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175782.988498:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175782.988505:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175782.988512:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175782.988521:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175782.988531:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175782.988537:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175782.988540:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175783.117244:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175783.117252:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175783.118151:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175783.118153:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175783.118402:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175783.118404:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175783.118618:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175783.118620:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175783.119040:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175783.119042:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1786175784.012481:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175784.012493:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175784.012496:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175784.012503:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175784.012511:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175784.012519:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175784.012526:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175784.012533:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175784.012541:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175785.036497:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175785.036525:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175785.036529:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175785.036538:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175785.036544:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175785.036551:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175785.036560:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175785.036566:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175785.036576:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175786.060489:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175786.060501:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175786.060509:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175786.060520:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175786.060528:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175786.060534:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175786.060538:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175786.060557:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175786.060566:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175786.700526:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175786.700535:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175786.700537:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175786.700542:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175786.700547:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175786.700551:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175786.700556:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175786.700560:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175786.700563:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175787.084476:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175787.084485:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175787.084489:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175787.084496:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175787.084502:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175787.084508:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175787.084514:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175787.084519:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175787.084525:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175787.724444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175787.724456:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175787.724458:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175787.724466:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175787.724472:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175787.724478:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175787.724482:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175787.724486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175787.724491:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.108457:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175788.108476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175788.108478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175788.108483:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175788.108487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175788.108495:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175788.108500:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175788.108505:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175788.108510:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175788.236564:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175788.236576:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175788.236579:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175788.236584:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175788.236589:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175788.236597:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10462 00000400:00000200:3.0:1786175788.236608:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175788.236618:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175788.236623:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175788.236628:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.236633:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175788.236637:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.236641:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175788.236648:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175788.236651:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175788.236654:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175788.236659:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175788.236663:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175788.236667:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175788.236673:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.236677:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175788.236680:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.236683:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175788.236688:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175788.236694:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175788.236698:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175788.236702:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175788.236706:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175788.236711:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.236714:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175788.236717:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175788.236729:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786175788.236733:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175788.236738:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175788.236739:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.236743:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175788.236746:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175788.236750:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175788.236759:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786175788.236766:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175788.236777:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235411200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175788.236788:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175788.236794:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175788.236799:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235411200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175788.236826:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175788.236830:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175788.236834:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175788.236835:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175788.236838:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175788.236841:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175788.236844:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10462 00000400:00000200:3.0:1786175788.236848:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175788.236852:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175788.236854:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175788.236857:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.236860:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175788.236876:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.236879:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175788.236882:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175788.236883:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175788.236885:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175788.236889:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175788.236893:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175788.236902:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175788.236914:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.236920:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175788.236931:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.236941:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175788.236967:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175788.236979:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175788.236991:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175788.237000:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175788.237011:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175788.237019:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.237027:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175788.237036:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175788.237048:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.237053:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175788.237064:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786175788.237074:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175788.237083:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175788.237087:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.237093:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175788.237097:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175788.237102:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175788.237112:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786175788.237122:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175788.237136:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235411328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175788.237155:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175788.237161:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175788.237172:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235411328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175788.237197:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175788.237209:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175788.237220:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175788.237224:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175788.237230:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175788.237235:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175788.237241:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10462 00000400:00000200:3.0:1786175788.237247:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175788.237251:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175788.237257:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175788.237267:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.237273:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175788.237281:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.237286:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175788.237290:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175788.237303:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175788.237307:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175788.237311:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175788.237316:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175788.237322:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175788.237330:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.237335:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1786175788.237337:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786175788.237341:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.237346:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:1.1:1786175788.237349:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000020:00000040:3.0:1786175788.237350:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175788.237392:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175788.237396:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175788.237399:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175788.237405:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175788.237409:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.237411:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175788.237413:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175788.237416:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.237418:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175788.237423:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786175788.237426:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175788.237429:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175788.237431:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.237436:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175788.237439:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175788.237444:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175788.237451:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786175788.237457:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175788.237464:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235411456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175788.237482:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175788.237487:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175788.237495:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235411456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175788.237508:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175788.237511:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175788.237517:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175788.237519:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175788.237526:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175788.237529:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175788.237536:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10462 00000400:00000200:3.0:1786175788.237547:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175788.237555:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175788.237561:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175788.237569:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.237573:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175788.237577:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.237582:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175788.237587:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175788.237589:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175788.237595:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175788.237600:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175788.237608:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175788.237619:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175788.237631:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.237641:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175788.237652:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.237662:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175788.237669:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175788.237679:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175788.237690:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175788.237699:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175788.237706:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175788.237714:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.237719:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175788.237725:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175788.237733:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.237741:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175788.237749:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786175788.237755:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175788.237764:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175788.237769:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.237775:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175788.237779:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175788.237785:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175788.237793:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786175788.237801:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175788.237815:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235411584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175788.237838:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175788.237844:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175788.237857:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235411584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175788.237875:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175788.237883:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175788.237896:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175788.237905:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175788.237913:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175788.237925:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175788.237935:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10462 00000400:00000200:3.0:1786175788.237975:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175788.237988:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175788.237992:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175788.238007:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.238018:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175788.238028:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.238038:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175788.238048:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175788.238054:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175788.238061:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175788.238069:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175788.238078:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175788.238086:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175788.238092:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.238097:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175788.238104:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175788.238112:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175788.238118:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175788.238126:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175788.238135:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175788.238140:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175788.238147:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175788.238156:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.238160:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175788.238167:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175788.238175:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.238183:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175788.238202:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786175788.238210:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175788.238214:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175788.238218:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175788.238223:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175788.238226:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175788.238230:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175788.238239:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786175788.238247:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175788.238261:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235411712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175788.238279:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175788.238284:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175788.238304:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235411712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175788.238345:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175788.238353:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10472) 00000800:00000200:1.1:1786175788.238497:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175788.238502:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175788.238945:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175788.238992:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175788.239445:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175788.239449:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175788.239924:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175788.239927:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1786175788.748478:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175788.748487:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175788.748489:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175788.748495:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175788.748499:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175788.748502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175788.748505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175788.748507:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175788.748511:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175789.772448:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175789.772460:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175789.772463:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175789.772471:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175789.772478:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175789.772484:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175789.772488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175789.772492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175789.772496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175793.356468:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175793.356481:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175793.356484:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175793.356489:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175793.356495:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175793.356505:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10467 00000400:00000200:3.0:1786175793.356518:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175793.356529:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175793.356534:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175793.356540:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.356545:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175793.356549:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.356553:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175793.356559:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175793.356563:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175793.356567:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175793.356572:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175793.356576:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175793.356580:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175793.356585:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.356588:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175793.356591:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.356594:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175793.356598:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175793.356602:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175793.356604:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175793.356608:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175793.356612:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175793.356616:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.356619:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175793.356622:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175793.356636:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786175793.356641:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175793.356645:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175793.356647:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.356652:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175793.356655:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175793.356661:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175793.356671:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786175793.356681:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175793.356692:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235411840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175793.356704:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175793.356709:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175793.356713:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235411840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175793.356742:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175793.356745:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175793.356749:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175793.356751:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175793.356752:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175793.356755:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175793.356757:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10467 00000400:00000200:3.0:1786175793.356761:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175793.356764:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175793.356767:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175793.356771:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.356774:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175793.356776:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.356780:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175793.356784:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175793.356785:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175793.356786:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175793.356788:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175793.356791:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175793.356794:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175793.356797:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.356798:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175793.356804:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.356808:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175793.356811:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175793.356825:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175793.356834:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175793.356843:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175793.356852:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175793.356864:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.356873:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175793.356882:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175793.356919:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.356922:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175793.356926:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786175793.356934:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175793.356945:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175793.356948:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.356952:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175793.356955:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175793.356959:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175793.356977:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786175793.356986:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175793.357009:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235411968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175793.357028:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175793.357032:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175793.357044:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235411968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175793.357058:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175793.357066:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175793.357073:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175793.357082:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175793.357088:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175793.357095:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175793.357101:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10467 00000400:00000200:3.0:1786175793.357114:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175793.357121:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175793.357139:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175793.357155:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.357162:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175793.357170:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.357181:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175793.357194:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175793.357199:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175793.357207:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175793.357217:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175793.357225:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175793.357230:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175793.357243:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.357258:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175793.357262:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.357269:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175793.357280:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175793.357284:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175793.357286:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175793.357292:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175793.357301:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175793.357308:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.357310:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175793.357312:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175793.357314:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.357316:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175793.357321:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786175793.357327:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175793.357341:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175793.357343:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.357346:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175793.357349:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175793.357351:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175793.357355:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786175793.357388:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175793.357394:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235412096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175793.357401:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175793.357403:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175793.357406:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235412096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175793.357415:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175793.357418:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175793.357422:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175793.357426:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175793.357428:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175793.357449:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175793.357464:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10467 00000400:00000200:3.0:1786175793.357486:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175793.357505:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175793.357515:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175793.357538:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.357548:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175793.357565:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.357580:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175793.357597:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175793.357601:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175793.357607:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175793.357613:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175793.357623:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175793.357633:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175793.357646:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.357650:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175793.357651:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.357653:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175793.357656:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175793.357659:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:1.1:1786175793.357672:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786175793.357673:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175793.357692:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1786175793.357693:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 02000000:00000001:3.0:1786175793.357698:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175793.357707:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.357716:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175793.357721:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175793.357727:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.357730:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175793.357756:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786175793.357766:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175793.357773:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175793.357786:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.357795:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175793.357799:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175793.357803:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175793.357812:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786175793.357825:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175793.357845:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235412224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175793.357856:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175793.357864:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175793.357891:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235412224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175793.357952:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175793.357958:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175793.357965:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175793.357968:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175793.357975:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175793.357988:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175793.358002:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10467 00000400:00000200:3.0:1786175793.358016:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175793.358023:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175793.358033:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175793.358039:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.358044:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175793.358050:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.358059:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175793.358077:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175793.358082:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175793.358100:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175793.358113:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175793.358126:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175793.358130:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175793.358141:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.358155:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175793.358166:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175793.358178:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175793.358185:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175793.358197:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175793.358218:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175793.358232:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175793.358238:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175793.358247:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.358251:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175793.358254:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175793.358262:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.358266:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175793.358273:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786175793.358281:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175793.358289:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175793.358294:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175793.358304:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175793.358311:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175793.358316:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175793.358331:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786175793.358341:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175793.358350:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235412352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175793.358364:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175793.358366:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175793.358371:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235412352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175793.358467:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175793.358477:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10477) 00000800:00000200:1.1:1786175793.358831:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175793.358835:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175793.359263:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175793.359268:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175793.359739:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175793.359743:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175793.360130:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175793.360133:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175798.477343:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175798.477353:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175798.478432:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175798.478435:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175798.478893:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175798.478899:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175798.479233:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175798.479236:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175798.479566:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175798.479569:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175803.597450:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175803.597461:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175803.598703:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175803.598706:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175803.599116:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175803.599120:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175803.599513:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175803.599516:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786175803.599918:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175803.599922:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175808.717298:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175808.717306:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175808.718162:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175808.718166:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175808.718564:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175808.718569:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175808.718854:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175808.718857:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786175808.719189:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175808.719193:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175813.837183:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175813.837190:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175813.838029:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175813.838033:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175813.838432:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175813.838436:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175813.838810:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175813.838814:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786175813.839298:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175813.839302:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1786175815.372518:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175815.372523:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175815.372526:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175815.372530:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175815.372534:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175815.372538:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175815.372543:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175815.372546:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175815.372550:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175816.396494:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175816.396505:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175816.396510:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175816.396518:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175816.396525:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175816.396530:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175816.396534:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175816.396537:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175816.396541:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175817.420443:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175817.420454:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175817.420457:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175817.420464:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175817.420469:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175817.420475:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175817.420479:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175817.420483:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175817.420487:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.444448:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175818.444455:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175818.444458:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175818.444462:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175818.444466:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175818.444469:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175818.444472:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175818.444474:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175818.444477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175818.956481:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175818.956493:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175818.956496:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175818.956501:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175818.956506:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175818.956529:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10493 00000400:00000200:3.0:1786175818.956542:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175818.956552:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175818.956557:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175818.956564:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.956572:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175818.956577:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.956581:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175818.956589:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175818.956592:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175818.956596:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175818.956602:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175818.956606:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175818.956610:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175818.956616:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.956620:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175818.956624:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.956627:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175818.956632:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175818.956638:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175818.956644:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175818.956649:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175818.956678:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175818.956705:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.956707:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175818.956711:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175818.956720:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786175818.956724:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175818.956729:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175818.956731:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.956735:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175818.956738:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175818.956743:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175818.956751:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786175818.956758:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175818.956769:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235415040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175818.956788:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175818.956794:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175818.956800:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235415040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175818.956822:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175818.956826:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175818.956830:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175818.956832:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175818.956834:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175818.956838:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175818.956841:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10493 00000400:00000200:3.0:1786175818.956846:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175818.956851:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175818.956855:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175818.956860:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.956866:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175818.956873:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.956880:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175818.956887:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175818.956890:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175818.956892:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175818.956895:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175818.956901:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175818.956905:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175818.956930:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.956936:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175818.956943:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.956950:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175818.956958:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175818.956995:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957000:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175818.957003:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175818.957011:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175818.957017:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.957023:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175818.957028:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175818.957035:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.957041:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175818.957048:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786175818.957053:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175818.957058:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175818.957061:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.957064:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175818.957068:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175818.957073:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175818.957081:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786175818.957089:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175818.957102:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235415168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175818.957116:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175818.957121:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175818.957130:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235415168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175818.957154:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175818.957160:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175818.957169:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175818.957172:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175818.957174:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175818.957178:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175818.957187:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10493 00000400:00000200:3.0:1786175818.957198:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175818.957208:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175818.957214:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175818.957221:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957228:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175818.957235:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:1.1:1786175818.957240:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786175818.957241:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175818.957247:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:1.1:1786175818.957247:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000020:00000040:3.0:1786175818.957250:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175818.957254:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175818.957260:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175818.957266:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175818.957272:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175818.957279:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957283:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175818.957290:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957294:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175818.957297:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175818.957303:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957310:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175818.957314:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175818.957320:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175818.957327:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.957332:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175818.957336:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175818.957343:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.957348:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175818.957354:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786175818.957359:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175818.957362:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175818.957365:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.957368:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175818.957370:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175818.957373:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175818.957378:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786175818.957382:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175818.957420:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235415296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175818.957440:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175818.957446:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175818.957456:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235415296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175818.957474:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175818.957478:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175818.957486:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175818.957492:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175818.957497:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175818.957504:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175818.957512:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10493 00000400:00000200:3.0:1786175818.957523:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175818.957534:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175818.957541:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175818.957549:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957556:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175818.957564:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957571:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175818.957578:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175818.957582:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175818.957586:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175818.957592:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175818.957599:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175818.957606:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175818.957615:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957621:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175818.957628:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957634:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175818.957641:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175818.957649:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957670:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175818.957675:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175818.957681:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175818.957687:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.957692:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175818.957696:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175818.957701:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.957708:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175818.957714:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786175818.957721:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175818.957728:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175818.957733:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.957739:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175818.957742:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175818.957746:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175818.957754:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786175818.957761:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175818.957774:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235415424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175818.957788:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175818.957794:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175818.957802:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235415424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175818.957822:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175818.957828:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175818.957840:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175818.957844:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175818.957849:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175818.957857:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175818.957865:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10493 00000400:00000200:3.0:1786175818.957877:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175818.957889:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175818.957910:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175818.957923:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957931:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175818.957939:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.957973:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175818.957979:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175818.957982:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175818.957984:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175818.957987:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175818.957993:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175818.957996:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175818.958004:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.958008:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175818.958014:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175818.958018:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:1.1:1786175818.958019:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786175818.958023:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000800:00000200:1.1:1786175818.958024:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00080000:3.0:1786175818.958030:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175818.958035:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175818.958038:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175818.958043:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175818.958048:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.958053:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175818.958058:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175818.958065:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.958069:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175818.958074:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786175818.958079:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175818.958084:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175818.958087:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175818.958090:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175818.958093:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175818.958096:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175818.958102:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786175818.958108:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175818.958116:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235415552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175818.958134:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175818.958139:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175818.958144:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235415552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175818.958191:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175818.958201:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10503) 00000800:00000200:1.1:1786175818.958453:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175818.958456:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175818.958873:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175818.958875:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175818.959184:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175818.959186:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1786175819.468477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175819.468487:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175819.468490:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175819.468496:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175819.468502:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175819.468507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175819.468510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175819.468514:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175819.468519:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175822.540700:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175822.540714:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175822.540718:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175822.540728:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175822.540736:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175822.540747:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175822.540752:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175822.540757:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175822.540763:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175824.077699:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175824.077714:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175824.078759:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175824.078762:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175824.079033:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175824.079035:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175824.079335:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175824.079338:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175824.079720:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175824.079723:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786175829.197344:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175829.197352:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175829.198477:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175829.198481:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175829.198929:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175829.198932:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175829.199356:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175829.199359:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175829.199707:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175829.199709:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175834.316989:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175834.316994:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175834.317638:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175834.317640:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175834.317856:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175834.317857:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175834.318037:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175834.318038:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786175834.318207:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175834.318209:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000020:00000001:3.0:1786175836.236650:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175836.236671:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175836.236676:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175838.284714:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175838.284719:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175838.284723:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786175839.437686:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175839.438068:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175839.438223:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175839.438410:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175839.438611:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:3.0:1786175840.332605:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175840.332610:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175840.332614:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175842.380846:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175842.380855:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175842.380862:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175844.428597:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175844.428601:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175844.428605:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786175844.556493:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175844.556503:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175844.556506:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175844.556509:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175844.556512:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175844.556519:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10519 00000400:00000200:3.0:1786175844.556526:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175844.556538:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175844.556540:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175844.556543:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.556546:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175844.556548:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.556549:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175844.556553:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175844.556554:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175844.556557:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175844.556561:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175844.556564:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175844.556566:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175844.556570:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.556572:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175844.556574:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.556576:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175844.556579:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175844.556582:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175844.556585:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175844.556589:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175844.556592:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175844.556595:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.556597:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175844.556600:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175844.556619:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786175844.556623:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175844.556627:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175844.556629:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.556633:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175844.556636:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175844.556640:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175844.556648:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786175844.556656:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175844.556666:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235418240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175844.556677:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175844.556682:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175844.556687:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235418240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175844.556710:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175844.556715:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175844.556719:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175844.556720:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175844.556722:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175844.556725:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175844.556728:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10519 00000400:00000200:3.0:1786175844.556733:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175844.556737:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175844.556740:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175844.556744:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.556747:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175844.556750:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.556754:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175844.556758:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175844.556762:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175844.556767:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175844.556773:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175844.556778:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175844.556780:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175844.556789:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.556794:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175844.556799:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.556804:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175844.556810:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175844.556816:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175844.556821:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175844.556826:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175844.556833:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175844.556839:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.556844:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175844.556867:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175844.556873:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.556875:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175844.556883:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786175844.556888:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175844.556894:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175844.556898:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.556903:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175844.556906:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175844.556908:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175844.556914:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786175844.556927:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175844.556938:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235418368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175844.556962:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175844.556967:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175844.556979:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235418368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175844.556996:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175844.557001:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175844.557005:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175844.557006:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175844.557007:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175844.557010:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175844.557015:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10519 00000400:00000200:3.0:1786175844.557023:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175844.557028:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175844.557031:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175844.557037:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.557041:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175844.557046:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.557053:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175844.557056:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175844.557066:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175844.557070:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175844.557074:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175844.557080:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175844.557085:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175844.557093:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.557100:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175844.557105:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.557110:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175844.557113:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175844.557118:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175844.557124:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175844.557129:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175844.557135:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175844.557141:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.557147:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175844.557150:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175844.557155:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.557157:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175844.557164:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000800:00000200:2.1:1786175844.557175:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175844.557179:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000020:00000040:3.0:1786175844.557250:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175844.557255:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175844.557258:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.557263:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175844.557266:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175844.557270:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175844.557276:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786175844.557282:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175844.557290:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235418496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175844.557316:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175844.557321:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175844.557327:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235418496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175844.557343:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175844.557348:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175844.557417:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175844.557421:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175844.557431:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175844.557437:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175844.557446:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10519 00000400:00000200:3.0:1786175844.557458:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175844.557466:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175844.557471:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175844.557478:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.557482:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175844.557488:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.557494:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175844.557501:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175844.557506:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175844.557510:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175844.557515:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175844.557520:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175844.557526:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175844.557535:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.557539:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175844.557546:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.557552:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175844.557559:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175844.557566:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175844.557572:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175844.557576:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175844.557582:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175844.557587:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.557592:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175844.557596:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175844.557602:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.557608:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175844.557615:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786175844.557622:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175844.557629:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175844.557633:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.557639:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175844.557642:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175844.557646:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175844.557652:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786175844.557656:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175844.557666:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235418624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175844.557698:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175844.557704:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175844.557906:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175844.557910:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000040:3.0:1786175844.558011:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235418624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175844.558031:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175844.558038:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175844.558046:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175844.558049:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175844.558052:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175844.558064:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175844.558084:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10519 00000400:00000200:3.0:1786175844.558095:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175844.558104:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175844.558110:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175844.558119:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.558125:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175844.558133:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.558139:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175844.558144:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175844.558148:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175844.558152:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175844.558154:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175844.558156:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175844.558159:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175844.558162:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.558164:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175844.558166:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175844.558171:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175844.558178:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175844.558184:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175844.558190:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175844.558195:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175844.558201:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175844.558206:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.558212:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175844.558216:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175844.558222:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.558227:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175844.558235:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786175844.558241:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175844.558248:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175844.558253:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175844.558260:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175844.558276:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175844.558282:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175844.558294:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786175844.558302:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175844.558314:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235418752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175844.558334:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175844.558340:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175844.558348:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235418752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175844.558430:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175844.558439:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10529) 00000800:00000200:2.1:1786175844.558511:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175844.558514:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175844.558848:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175844.558861:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175844.559135:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175844.559137:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000020:00000001:3.0:1786175846.476539:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175846.476543:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175846.476547:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175847.116503:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175847.116511:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175847.116513:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175847.116519:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175847.116522:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175847.116526:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175847.116528:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175847.116531:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175847.116533:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175848.524623:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175848.524627:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175848.524630:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786175849.677052:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175849.677058:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175849.677820:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175849.677824:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175849.678104:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175849.678105:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175849.678341:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175849.678343:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175849.678585:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175849.678587:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000020:00000001:3.0:1786175850.572537:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175850.572541:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175850.572544:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175852.236627:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175852.236637:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175852.236640:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175852.236647:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175852.236652:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175852.236656:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175852.236660:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175852.236663:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175852.236667:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175852.620544:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175852.620548:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175852.620551:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175853.260428:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175853.260435:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175853.260437:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175853.260442:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175853.260446:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175853.260449:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175853.260452:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175853.260454:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175853.260457:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.284440:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175854.284448:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175854.284451:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175854.284456:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175854.284460:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175854.284463:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175854.284466:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175854.284468:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175854.284471:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175854.668514:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175854.668520:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175854.668523:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786175854.796482:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175854.796491:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175854.796493:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175854.796497:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175854.796500:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175854.796507:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10529 00000400:00000200:3.0:1786175854.796515:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175854.796521:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175854.796523:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175854.796526:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796528:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175854.796530:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796532:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175854.796536:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175854.796537:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175854.796539:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175854.796542:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175854.796544:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175854.796545:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175854.796547:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796548:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175854.796550:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796551:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175854.796554:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175854.796567:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796568:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175854.796571:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175854.796573:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175854.796575:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.796577:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175854.796579:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175854.796587:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786175854.796590:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175854.796593:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175854.796594:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.796597:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175854.796599:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175854.796602:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175854.796609:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786175854.796614:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175854.796622:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235419520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175854.796631:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175854.796635:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175854.796639:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235419520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175854.796656:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175854.796659:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175854.796662:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175854.796663:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175854.796664:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175854.796666:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175854.796668:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10529 00000400:00000200:3.0:1786175854.796670:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175854.796673:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175854.796674:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175854.796676:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796678:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175854.796679:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796681:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175854.796682:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175854.796683:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175854.796684:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175854.796685:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175854.796686:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175854.796688:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175854.796690:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796692:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175854.796694:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796696:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175854.796698:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175854.796701:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796703:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175854.796705:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175854.796709:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175854.796711:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.796712:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175854.796716:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175854.796720:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.796722:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175854.796726:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786175854.796730:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175854.796733:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175854.796734:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.796738:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175854.796740:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175854.796742:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175854.796747:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786175854.796752:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175854.796759:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235419648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175854.796771:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175854.796775:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175854.796778:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235419648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175854.796791:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175854.796795:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175854.796804:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175854.796806:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175854.796809:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175854.796814:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175854.796820:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10529 00000400:00000200:3.0:1786175854.796824:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175854.796826:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175854.796831:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175854.796836:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796838:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175854.796843:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796846:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175854.796848:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175854.796849:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175854.796858:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175854.796861:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175854.796863:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175854.796865:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175854.796871:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796872:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175854.796874:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796876:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175854.796882:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175854.796887:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175854.796891:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175854.796894:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175854.796896:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175854.796901:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.796905:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175854.796908:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175854.796911:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.796924:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175854.796930:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786175854.796933:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175854.796938:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175854.796940:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.796943:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175854.796944:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175854.796946:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175854.796951:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786175854.796954:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175854.796960:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235419776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175854.796970:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175854.796972:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175854.796978:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235419776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175854.796990:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175854.796995:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175854.797000:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175854.797002:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175854.797005:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175854.797009:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175854.797015:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10529 00000400:00000200:3.0:1786175854.797020:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175854.797029:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175854.797033:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175854.797039:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.797043:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175854.797047:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.797050:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175854.797055:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175854.797056:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175854.797059:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175854.797062:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175854.797068:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175854.797070:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175854.797076:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.797081:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1786175854.797081:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786175854.797085:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:2.1:1786175854.797087:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1786175854.797088:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175854.797092:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175854.797097:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175854.797101:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175854.797105:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175854.797107:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175854.797109:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.797113:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175854.797117:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175854.797122:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.797124:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175854.797127:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786175854.797132:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175854.797135:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175854.797139:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.797140:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175854.797143:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175854.797144:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175854.797151:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786175854.797155:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175854.797163:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235419904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175854.797174:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175854.797177:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175854.797184:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235419904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175854.797203:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175854.797206:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175854.797219:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175854.797221:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175854.797222:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175854.797227:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175854.797231:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10529 00000400:00000200:3.0:1786175854.797239:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175854.797243:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175854.797244:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175854.797249:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.797250:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175854.797252:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.797256:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175854.797258:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175854.797259:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175854.797260:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175854.797262:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175854.797267:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175854.797269:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175854.797274:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.797278:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175854.797281:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175854.797285:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175854.797291:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175854.797295:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175854.797300:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175854.797317:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175854.797320:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175854.797324:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.797328:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175854.797331:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175854.797346:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.797349:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175854.797354:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786175854.797379:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175854.797381:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175854.797382:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175854.797384:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175854.797385:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175854.797386:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175854.797389:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786175854.797393:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175854.797401:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235420032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175854.797417:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175854.797420:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175854.797427:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235420032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175854.797454:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175854.797459:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10539) 00000800:00000200:2.1:1786175854.797670:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175854.797672:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175854.797910:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175854.797911:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175854.798108:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175854.798110:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786175854.798296:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175854.798298:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000020:00000001:3.0:1786175856.716596:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175856.716606:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175856.716613:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175858.764546:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175858.764551:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175858.764555:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786175859.917052:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175859.917060:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175859.917764:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175859.917767:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175859.918085:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175859.918088:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175859.918417:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175859.918419:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175859.918786:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175859.918790:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000020:00000001:3.0:1786175860.812526:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175860.812530:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175860.812534:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175862.476468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175862.476475:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175862.476477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175862.476482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175862.476486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175862.476488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175862.476490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175862.476492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175862.476494:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175862.860534:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175862.860537:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175862.860540:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175863.500449:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175863.500456:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175863.500458:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175863.500463:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175863.500467:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175863.500470:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175863.500473:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175863.500476:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175863.500478:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175864.524457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175864.524467:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175864.524469:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175864.524475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175864.524481:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175864.524486:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175864.524490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175864.524494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175864.524500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175864.908609:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175864.908616:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175864.908623:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786175865.037466:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175865.037473:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175865.038833:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175865.038838:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175865.039248:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175865.039251:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175865.039648:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175865.039653:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786175865.040180:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175865.040183:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00000001:3.0:1786175865.548422:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175865.548429:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175865.548431:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175865.548443:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175865.548447:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175865.548450:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175865.548454:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175865.548456:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175865.548458:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175866.572500:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175866.572513:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175866.572517:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175866.572526:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175866.572533:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175866.572539:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175866.572545:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175866.572550:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175866.572555:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175866.956503:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175866.956507:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175866.956509:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175867.596458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175867.596466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175867.596468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175867.596473:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175867.596477:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175867.596480:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175867.596482:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175867.596484:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175867.596486:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175868.620445:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175868.620454:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175868.620456:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175868.620462:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175868.620467:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175868.620471:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175868.620474:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175868.620477:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175868.620481:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175869.004631:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175869.004635:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175869.004640:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175869.644431:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175869.644438:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175869.644440:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175869.644445:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175869.644448:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175869.644451:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175869.644453:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175869.644456:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175869.644458:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175870.157285:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175870.157293:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175870.158194:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175870.158213:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175870.158509:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175870.158512:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175870.158741:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175870.158744:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786175870.159149:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175870.159151:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000100:00000001:3.0:1786175870.668445:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175870.668453:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175870.668455:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175870.668462:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175870.668465:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175870.668469:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175870.668471:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175870.668474:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175870.668477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175871.052574:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175871.052580:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175871.052586:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175871.692503:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175871.692512:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175871.692514:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175871.692520:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175871.692524:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175871.692528:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175871.692531:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175871.692534:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175871.692537:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175872.716522:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175872.716532:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175872.716535:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175872.716542:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175872.716547:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175872.716553:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175872.716558:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175872.716561:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175872.716566:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175873.100546:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175873.100550:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175873.100552:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175873.740454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175873.740464:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175873.740467:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175873.740474:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175873.740479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175873.740483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175873.740486:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175873.740489:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175873.740493:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175874.764458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175874.764466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175874.764469:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175874.764474:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175874.764478:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175874.764481:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175874.764484:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175874.764486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175874.764489:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175875.148632:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175875.148638:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175875.148642:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786175875.277131:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175875.277136:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786175875.277729:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175875.277731:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786175875.277926:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175875.277928:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786175875.278136:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175875.278138:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786175875.278335:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175875.278337:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000100:00000001:3.0:1786175875.788502:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175875.788512:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175875.788514:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175875.788521:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175875.788527:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175875.788532:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175875.788535:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175875.788538:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175875.788541:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175876.812545:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175876.812553:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175876.812555:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175876.812561:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175876.812567:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175876.812572:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175876.812576:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175876.812579:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175876.812582:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175877.196661:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175877.196666:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175877.196670:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175877.836468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175877.836478:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175877.836480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175877.836486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175877.836492:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175877.836495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175877.836498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175877.836501:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175877.836503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175878.860420:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175878.860430:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175878.860433:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175878.860440:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175878.860446:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175878.860451:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175878.860455:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175878.860459:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175878.860463:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175879.244518:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175879.244522:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175879.244525:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175879.884417:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175879.884424:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175879.884426:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175879.884431:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175879.884463:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175879.884470:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175879.884474:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175879.884479:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175879.884484:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175880.396977:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175880.396984:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786175880.397602:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175880.397603:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786175880.397788:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175880.397790:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786175880.397969:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175880.397970:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786175880.398183:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175880.398185:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000100:00000001:3.0:1786175880.908406:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175880.908413:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175880.908416:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175880.908420:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175880.908425:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175880.908428:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175880.908430:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175880.908433:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175880.908436:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175881.292514:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175881.292518:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175881.292521:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175883.340562:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175883.340566:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175883.340569:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175885.388417:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175885.388422:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175885.388426:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175885.388430:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175885.388436:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175885.388440:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175885.388442:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175885.388444:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175885.388446:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175885.516963:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175885.516968:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175885.517532:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175885.517534:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175885.517731:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175885.517733:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175885.517927:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175885.517930:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175885.518151:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175885.518153:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1786175886.412453:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175886.412462:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175886.412465:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175886.412474:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175886.412479:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175886.412484:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175886.412489:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175886.412493:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175886.412499:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175887.436444:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175887.436478:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175887.436482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175887.436489:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175887.436496:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175887.436501:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175887.436508:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175887.436514:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175887.436518:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175888.460465:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175888.460475:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175888.460478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175888.460484:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175888.460489:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175888.460504:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175888.460509:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175888.460512:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175888.460517:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175889.484442:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175889.484451:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175889.484453:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175889.484458:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175889.484461:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175889.484464:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175889.484467:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175889.484470:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175889.484473:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175890.508486:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175890.508494:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175890.508497:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175890.508502:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175890.508507:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175890.508522:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175890.508525:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175890.508527:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175890.508530:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175890.637565:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175890.637571:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175890.638583:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175890.638586:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175890.638988:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175890.638991:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175890.639317:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175890.639320:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175890.639688:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175890.639691:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1786175891.532458:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175891.532465:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175891.532467:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175891.532472:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175891.532476:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175891.532480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175891.532484:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175891.532487:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175891.532492:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175892.556453:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175892.556464:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175892.556468:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175892.556475:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175892.556481:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175892.556486:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175892.556490:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175892.556493:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175892.556497:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175893.580448:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175893.580458:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175893.580460:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175893.580468:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175893.580473:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175893.580479:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175893.580484:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175893.580487:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175893.580492:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175894.604500:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175894.604510:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175894.604514:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175894.604529:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175894.604536:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175894.604546:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175894.604551:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175894.604561:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175894.604572:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175895.628440:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175895.628449:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175895.628452:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175895.628458:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175895.628463:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175895.628467:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175895.628470:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175895.628474:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175895.628477:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786175895.757561:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175895.757568:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175895.758876:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175895.758897:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175895.759280:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175895.759282:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175895.759694:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175895.759697:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175895.760241:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175895.760245:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1786175896.652467:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175896.652477:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175896.652497:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175896.652505:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175896.652510:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175896.652514:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175896.652519:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175896.652525:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175896.652533:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175897.676416:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175897.676424:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175897.676426:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175897.676431:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175897.676437:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175897.676441:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175897.676446:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175897.676450:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175897.676456:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175898.700462:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175898.700471:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175898.700476:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175898.700482:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175898.700507:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175898.700514:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175898.700522:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175898.700527:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175898.700532:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175899.724464:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175899.724473:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175899.724476:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175899.724483:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175899.724487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175899.724493:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175899.724496:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175899.724500:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175899.724510:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175900.748454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175900.748461:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175900.748464:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175900.748470:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175900.748475:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175900.748480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175900.748483:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175900.748486:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175900.748489:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175900.877023:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175900.877030:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175900.877661:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175900.877663:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175900.877901:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175900.877903:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175900.878085:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175900.878087:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786175900.878383:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175900.878385:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1786175901.772456:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175901.772464:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175901.772467:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175901.772471:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175901.772476:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175901.772480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175901.772484:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175901.772487:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175901.772489:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175902.796440:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175902.796448:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175902.796450:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175902.796456:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175902.796461:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175902.796466:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175902.796471:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175902.796476:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175902.796480:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175903.820469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175903.820482:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175903.820486:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175903.820493:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175903.820499:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175903.820505:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175903.820510:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175903.820531:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175903.820534:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175903.820646:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175903.820652:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175903.820654:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175904.844494:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175904.844503:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175904.844509:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175904.844522:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175904.844531:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175904.844536:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175904.844540:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175904.844544:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175904.844549:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175905.868451:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175905.868459:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175905.868463:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175905.868467:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175905.868474:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175905.868481:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175905.868487:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175905.868492:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175905.868500:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175905.997030:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175905.997512:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175905.997672:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175905.997853:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175905.998001:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786175906.892482:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175906.892489:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175906.892493:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175906.892502:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175906.892508:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175906.892512:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175906.892516:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175906.892519:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175906.892525:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175907.916409:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175907.916416:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175907.916418:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175907.916423:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175907.916426:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175907.916429:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175907.916431:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175907.916434:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175907.916440:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175908.940460:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175908.940469:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175908.940472:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175908.940477:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175908.940480:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175908.940485:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175908.940488:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175908.940490:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175908.940494:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1786175909.068602:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:3.0:1786175909.068640:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1786175909.068648:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175909.068651:0:2381:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:3.0:1786175909.068656:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175909.068660:0:2381:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1786175909.068666:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175909.068670:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:3.0:1786175909.068692:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175909.068698:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175909.068705:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e3b8 00000400:00000010:3.0:1786175909.068708:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e3b8. 00000100:00000001:3.0:1786175909.068713:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175909.068722:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735c00 x1872930235425920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786175930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175909.068749:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175909.068754:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235425920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786175930 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175909.068763:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175909.068765:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175909.068776:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175909.068792:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:3.0:1786175909.068809:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175909.068813:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175909.068834:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:3.0:1786175909.068837:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:3.0:1786175909.068840:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175909.068846:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31737b80 x1872930235426048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175909.068870:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175909.068877:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235426048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175930 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175909.068893:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175909.068895:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175909.068905:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175909.068909:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:3.0:1786175909.068913:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175909.068919:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175909.068942:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:3.0:1786175909.068948:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:3.0:1786175909.068955:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175909.068964:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735500 x1872930235426176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175909.068997:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175909.069004:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235426176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175930 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175909.069017:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175909.069022:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175909.069032:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175909.069038:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:3.0:1786175909.069045:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175909.069062:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175909.069071:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e440 00000400:00000010:3.0:1786175909.069076:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e440. 00000100:00000001:3.0:1786175909.069082:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175909.069092:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735880 x1872930235426304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175909.069106:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175909.069122:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735880 x1872930235426304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175930 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175909.069136:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175909.069141:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175909.069151:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175909.069155:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:3.0:1786175909.069161:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175909.069173:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175909.069185:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:3.0:1786175909.069188:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:3.0:1786175909.069190:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175909.069198:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31734700 x1872930235426432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175909.069216:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175909.069224:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734700 x1872930235426432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175930 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175909.069247:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175909.069251:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1786175909.964438:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175909.964447:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175909.964449:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175909.964455:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175909.964460:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175909.964465:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175909.964468:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175909.964470:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175909.964473:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175910.988467:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175910.988477:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175910.988481:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175910.988489:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175910.988495:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175910.988501:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175910.988511:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175910.988516:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175910.988520:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175911.118010:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175911.118028:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175911.118787:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175911.118791:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175911.119335:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175911.119338:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175911.119656:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175911.119658:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786175911.119902:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175911.119904:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786175912.012459:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175912.012466:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175912.012468:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175912.012474:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175912.012480:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175912.012484:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175912.012488:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175912.012491:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175912.012494:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175913.036429:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175913.036436:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175913.036438:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175913.036445:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175913.036450:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175913.036454:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175913.036458:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175913.036462:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175913.036464:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175914.060420:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175914.060428:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175914.060431:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175914.060435:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175914.060439:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175914.060442:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175914.060445:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175914.060448:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175914.060451:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175915.084432:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175915.084438:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175915.084442:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175915.084450:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175915.084455:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175915.084460:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175915.084474:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175915.084478:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175915.084482:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175916.108446:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175916.108455:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175916.108459:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175916.108466:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175916.108472:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175916.108477:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175916.108483:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175916.108488:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175916.108493:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175916.237111:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175916.237116:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175916.237891:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175916.237896:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175916.238124:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175916.238126:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175916.238310:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175916.238312:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786175916.238618:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175916.238620:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1786175917.132469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175917.132479:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175917.132484:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175917.132495:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175917.132504:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175917.132510:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175917.132516:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175917.132521:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175917.132524:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175918.156462:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175918.156470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175918.156474:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175918.156480:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175918.156484:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175918.156489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175918.156493:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175918.156496:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175918.156500:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175919.180429:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175919.180436:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175919.180438:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175919.180445:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175919.180449:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175919.180453:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175919.180457:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175919.180461:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175919.180464:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175920.204456:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175920.204464:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175920.204468:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175920.204476:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175920.204483:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175920.204489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175920.204493:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175920.204497:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175920.204501:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175921.228476:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175921.228481:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175921.228485:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175921.228489:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175921.228495:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175921.228499:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175921.228510:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175921.228512:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175921.228515:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175921.357057:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175921.357063:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175921.357693:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175921.357695:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175921.357895:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175921.357897:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175921.358112:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175921.358114:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175921.358322:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175921.358324:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1786175922.252449:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175922.252456:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175922.252460:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175922.252466:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175922.252470:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175922.252475:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175922.252481:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175922.252483:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175922.252486:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175923.276432:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175923.276439:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175923.276441:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175923.276448:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175923.276453:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175923.276457:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175923.276460:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175923.276464:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175923.276468:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175924.300479:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175924.300488:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175924.300492:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175924.300498:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175924.300506:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175924.300512:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175924.300516:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175924.300518:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175924.300521:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175925.324421:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175925.324428:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175925.324432:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175925.324439:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175925.324445:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175925.324450:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175925.324453:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175925.324455:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175925.324458:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175926.348441:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175926.348450:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175926.348452:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175926.348473:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175926.348489:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175926.348494:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175926.348497:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175926.348501:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175926.348506:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175926.476995:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175926.477003:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175926.477784:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175926.477786:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175926.478040:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175926.478042:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175926.478244:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175926.478246:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786175926.478539:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175926.478542:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1786175927.372465:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175927.372475:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175927.372478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175927.372485:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175927.372491:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175927.372496:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175927.372500:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175927.372504:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175927.372509:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175928.012564:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175928.012573:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175928.012575:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175928.012579:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175928.012583:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175928.012587:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175928.012590:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175928.012593:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175928.012608:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175928.396443:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175928.396450:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175928.396451:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175928.396456:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175928.396461:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175928.396464:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175928.396469:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175928.396473:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175928.396480:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175929.036454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175929.036472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175929.036474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175929.036479:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175929.036482:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175929.036486:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175929.036488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175929.036490:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175929.036494:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175929.420412:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175929.420419:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175929.420421:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175929.420425:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175929.420429:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175929.420434:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175929.420439:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175929.420442:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175929.420446:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175930.060475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175930.060483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175930.060485:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175930.060490:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175930.060494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175930.060497:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175930.060500:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175930.060503:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175930.060506:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175930.444457:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175930.444464:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175930.444465:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175930.444486:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175930.444489:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175930.444492:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175930.444495:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175930.444498:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175930.444501:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175931.084467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175931.084473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175931.084475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175931.084480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175931.084483:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175931.084494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175931.084497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175931.084499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175931.084502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175931.468445:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175931.468458:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175931.468461:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175931.468469:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786175931.468474:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175931.468478:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175931.468482:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175931.468486:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175931.468491:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175931.597834:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175931.597846:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175931.599111:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175931.599116:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175931.599746:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175931.599751:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175931.600327:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175931.600332:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786175931.600983:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175931.600988:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1786175932.108450:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175932.108460:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175932.108463:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175932.108470:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175932.108474:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175932.108479:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175932.108482:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175932.108485:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175932.108488:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175932.492590:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175932.492595:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175932.492599:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786175933.132493:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786175933.132504:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786175933.132508:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175933.132515:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786175933.132519:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786175933.132523:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175933.132526:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786175933.132528:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786175933.132533:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786175934.540602:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175934.540606:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175934.540610:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175936.588600:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175936.588607:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175936.588612:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786175936.717095:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175936.717101:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175936.717737:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175936.717738:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175936.717974:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175936.717976:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175936.718167:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175936.718169:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786175936.718352:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175936.718353:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000020:00000001:3.0:1786175938.636747:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175938.636757:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175938.636764:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175940.684580:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175940.684588:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175940.684593:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786175941.837059:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175941.837065:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175941.837736:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175941.837740:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175941.837964:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175941.837966:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175941.838147:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175941.838148:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786175941.838427:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175941.838428:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000020:00000001:3.0:1786175942.732656:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175942.732664:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175942.732670:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175944.780707:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175944.780714:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175944.780719:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175946.828646:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175946.828650:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175946.828655:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786175946.957274:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175946.957303:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175946.958322:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175946.958324:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175946.958583:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175946.958585:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175946.958863:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175946.958867:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786175946.959142:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175946.959144:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000020:00000001:3.0:1786175948.876630:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175948.876636:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175948.876641:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175950.924582:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175950.924588:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175950.924593:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786175952.077150:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175952.077159:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175952.077987:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175952.077990:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175952.078290:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175952.078292:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175952.078727:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175952.078731:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786175952.079064:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175952.079066:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000020:00000001:3.0:1786175952.972562:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175952.972569:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175952.972573:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175955.020715:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175955.020721:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175955.020726:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175957.068636:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175957.068689:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175957.068694:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786175957.196466:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175957.196517:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786175957.196519:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175957.196522:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175957.196526:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175957.196532:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10631 00000400:00000200:3.0:1786175957.196540:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175957.196544:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175957.196547:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175957.196550:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196552:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175957.196554:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196556:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175957.196559:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175957.196561:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786175957.196563:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175957.196566:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786175957.196567:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786175957.196569:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786175957.196571:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196573:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175957.196574:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196575:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175957.196579:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786175957.196581:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196583:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175957.196585:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175957.196588:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175957.196590:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.196591:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786175957.196593:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175957.196601:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786175957.196604:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786175957.196608:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175957.196609:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.196612:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175957.196613:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175957.196617:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175957.196623:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786175957.196628:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175957.196636:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235432320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175957.196644:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175957.196648:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175957.196651:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235432320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175957.196668:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175957.196670:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175957.196673:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786175957.196674:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175957.196675:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175957.196677:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175957.196679:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10631 00000400:00000200:3.0:1786175957.196682:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175957.196686:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175957.196689:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175957.196693:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196696:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175957.196703:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196708:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175957.196714:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175957.196716:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786175957.196720:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175957.196725:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786175957.196730:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786175957.196737:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786175957.196747:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196751:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175957.196754:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196755:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175957.196758:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786175957.196764:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196768:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175957.196771:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175957.196776:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175957.196780:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.196783:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175957.196789:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175957.196792:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.196797:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175957.196803:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786175957.196806:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175957.196812:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175957.196813:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.196815:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175957.196817:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175957.196820:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175957.196825:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786175957.196828:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175957.196836:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235432448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175957.196848:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175957.196869:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175957.196878:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235432448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175957.196896:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175957.196901:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175957.196907:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786175957.196909:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175957.196912:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175957.196920:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175957.196923:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10631 00000400:00000200:3.0:1786175957.196929:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175957.196936:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175957.196941:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175957.196949:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196954:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175957.196958:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.196962:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175957.196966:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175957.196971:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786175957.196975:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175957.196979:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786175957.196986:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786175957.196992:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786175957.196998:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197004:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175957.197026:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197029:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175957.197032:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786175957.197037:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197040:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175957.197044:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175957.197046:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175957.197049:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.197051:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175957.197054:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786175957.197057:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.197062:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786175957.197066:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000010:3.0:1786175957.197068:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000800:00000200:1.1:1786175957.197070:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000020:00000040:3.0:1786175957.197071:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786175957.197075:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175957.197077:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.197081:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175957.197084:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175957.197086:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175957.197090:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786175957.197096:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175957.197103:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235432576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175957.197111:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175957.197115:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175957.197124:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235432576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175957.197132:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175957.197137:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175957.197146:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786175957.197149:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175957.197151:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175957.197155:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175957.197160:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10631 00000400:00000200:3.0:1786175957.197164:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175957.197171:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175957.197175:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175957.197180:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197184:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175957.197186:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197192:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175957.197193:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175957.197194:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786175957.197195:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175957.197198:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786175957.197200:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786175957.197202:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786175957.197204:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197206:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175957.197210:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197214:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175957.197217:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786175957.197221:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197225:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175957.197228:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175957.197233:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175957.197237:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.197242:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175957.197245:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175957.197265:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.197267:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175957.197272:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786175957.197280:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175957.197286:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175957.197290:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.197293:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175957.197297:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175957.197300:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175957.197306:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786175957.197311:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175957.197335:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235432704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175957.197350:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175957.197354:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175957.197397:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235432704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175957.197411:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786175957.197416:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786175957.197423:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786175957.197425:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786175957.197428:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786175957.197432:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786175957.197438:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10631 00000400:00000200:3.0:1786175957.197445:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175957.197450:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786175957.197455:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786175957.197461:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197466:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175957.197468:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197471:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786175957.197475:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786175957.197478:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786175957.197480:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786175957.197483:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786175957.197486:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786175957.197489:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786175957.197495:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197498:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786175957.197502:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197505:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786175957.197508:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786175957.197514:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786175957.197518:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786175957.197522:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786175957.197526:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786175957.197529:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.197531:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786175957.197535:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786175957.197541:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.197543:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786175957.197546:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786175957.197550:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786175957.197554:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786175957.197556:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175957.197559:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786175957.197562:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786175957.197564:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786175957.197572:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786175957.197577:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786175957.197586:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235432832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786175957.197608:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786175957.197613:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175957.197620:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235432832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175957.197631:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786175957.197634:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10641) 00000800:00000200:1.1:1786175957.197735:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175957.197737:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175957.197972:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175957.197974:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175957.198421:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175957.198424:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786175957.198800:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175957.198803:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000020:00000001:3.0:1786175959.116581:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175959.116584:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175959.116587:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175961.164592:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175961.164597:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175961.164601:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786175962.317604:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175962.317673:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175962.318823:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175962.318828:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175962.319253:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175962.319256:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175962.319740:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175962.319744:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786175962.320116:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175962.320121:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000020:00000001:3.0:1786175963.212580:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175963.212585:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175963.212589:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175965.260486:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175965.260491:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175965.260495:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175967.308603:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175967.308609:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175967.308613:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786175967.437204:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175967.437210:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175967.438114:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175967.438118:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175967.438502:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175967.438504:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175967.438747:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175967.438750:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786175967.439091:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175967.439095:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000020:00000001:3.0:1786175969.356600:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175969.356603:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175969.356606:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175970.380601:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175970.380615:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175970.380619:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175971.404663:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175971.404669:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175971.404674:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175972.428644:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175972.428648:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175972.428651:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786175972.557598:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175972.558400:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175972.559105:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175972.559825:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175972.560083:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:3.0:1786175973.452727:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175973.452734:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175973.452738:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175974.476647:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175974.476664:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175974.476667:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175975.500668:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175975.500672:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175975.500675:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1786175975.628703:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:3.0:1786175975.628765:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1786175975.628775:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175975.628779:0:2380:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:3.0:1786175975.628785:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786175975.628789:0:2380:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1786175975.628794:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175975.628798:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:3.0:1786175975.628804:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175975.628808:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175975.628815:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e220 00000400:00000010:3.0:1786175975.628818:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e220. 00000100:00000001:3.0:1786175975.628825:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175975.628836:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31734e00 x1872930235434240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786175997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175975.628878:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175975.628885:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734e00 x1872930235434240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786175997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175975.628897:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175975.628900:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175975.628916:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175975.628920:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:3.0:1786175975.628926:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175975.628931:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175975.628936:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e7f8 00000400:00000010:3.0:1786175975.628940:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e7f8. 00000100:00000001:3.0:1786175975.628944:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175975.628952:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31737100 x1872930235434368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175975.628971:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175975.628985:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235434368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175975.629031:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175975.629038:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175975.629051:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175975.629059:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:3.0:1786175975.629066:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175975.629075:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175975.629106:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ed48 00000400:00000010:3.0:1786175975.629113:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ed48. 00000100:00000001:3.0:1786175975.629122:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175975.629132:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31736a00 x1872930235434496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175975.629185:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175975.629197:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235434496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786175997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175975.629215:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175975.629221:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175975.629238:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175975.629246:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:3.0:1786175975.629254:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175975.629266:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175975.629277:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:3.0:1786175975.629282:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:3.0:1786175975.629291:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175975.629302:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735180 x1872930235434624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175975.629348:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175975.629410:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735180 x1872930235434624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175975.629423:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175975.629430:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786175975.629444:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786175975.629452:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:3.0:1786175975.629458:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786175975.629470:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786175975.629477:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:3.0:1786175975.629483:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:3.0:1786175975.629492:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786175975.629504:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31736680 x1872930235434752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175975.629523:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786175975.629536:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235434752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786175997 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786175975.629553:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786175975.629560:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1786175976.524615:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175976.524624:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175976.524632:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175977.548666:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175977.548669:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175977.548672:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786175977.677451:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175977.677461:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175977.678728:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175977.678732:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175977.679230:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175977.679234:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175977.679616:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175977.679619:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786175977.680023:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175977.680026:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000020:00000001:3.0:1786175978.572691:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175978.572697:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175978.572702:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175979.596651:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175979.596660:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175979.596667:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175980.620629:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175980.620635:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175980.620640:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175981.644672:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175981.644677:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175981.644682:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175982.668629:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175982.668635:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175982.668641:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786175982.797642:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175982.797651:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175982.798870:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175982.798874:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175982.799402:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175982.799405:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175982.799826:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175982.799829:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786175982.800303:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786175982.800307:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000020:00000001:3.0:1786175983.692538:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175983.692542:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175983.692546:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175984.716665:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175984.716671:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175984.716677:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175985.740691:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175985.740697:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175985.740702:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175986.764707:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175986.764717:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175986.764724:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175987.788691:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175987.788695:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175987.788698:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786175987.917504:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175987.917513:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175987.918444:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175987.918446:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175987.918769:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175987.918771:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175987.919121:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175987.919123:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786175987.919544:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175987.919547:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000020:00000001:3.0:1786175988.812650:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175988.812656:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175988.812661:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175989.836695:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175989.836701:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175989.836706:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175990.860720:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175990.860727:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175990.860731:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175991.884697:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175991.884704:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175991.884709:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175992.908523:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175992.908527:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175992.908531:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786175993.037694:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175993.037710:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175993.038565:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175993.038568:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175993.039080:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175993.039083:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175993.039477:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175993.039480:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786175993.039845:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175993.039848:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000020:00000001:3.0:1786175993.932704:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175993.932711:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175993.932717:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175994.956638:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175994.956650:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175994.956658:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175995.980684:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175995.980690:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175995.980695:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175997.004572:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175997.004582:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175997.004589:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786175998.028690:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175998.028696:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175998.028701:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786175998.157660:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175998.157670:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175998.159192:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175998.159196:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175998.159676:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175998.159681:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175998.160129:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175998.160132:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786175998.160732:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786175998.160736:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000020:00000001:3.0:1786175999.052700:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786175999.052707:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786175999.052713:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176000.076647:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176000.076654:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176000.076661:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176001.100729:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176001.100735:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176001.100741:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176002.124546:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176002.124549:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176002.124552:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176003.148671:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176003.148676:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176003.148681:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176003.277040:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176003.277048:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786176003.277672:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176003.277674:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786176003.277862:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176003.277864:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786176003.278093:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176003.278095:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786176003.278369:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176003.278371:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000020:00000001:3.0:1786176004.172547:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176004.172551:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176004.172554:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176005.196582:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176005.196587:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176005.196591:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176006.220738:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176006.220744:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176006.220749:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.0:1786176006.796475:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:3.0:1786176006.796485:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:3.0:1786176006.796491:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000020:00000001:3.0:1786176007.244600:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176007.244606:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176007.244610:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176008.268718:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176008.268737:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176008.268742:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176008.396474:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176008.396486:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176008.396489:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176008.396494:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176008.396499:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176008.396507:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10682 00000400:00000200:3.0:1786176008.396517:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176008.396524:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176008.396528:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176008.396532:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.396536:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176008.396539:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.396542:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176008.396547:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176008.396549:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176008.396552:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176008.396556:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176008.396558:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176008.396561:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176008.396565:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.396621:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176008.396624:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.396626:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176008.396630:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176008.396634:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176008.396637:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176008.396640:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176008.396644:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176008.396682:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.396685:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176008.396688:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176008.396701:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786176008.396705:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176008.396711:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176008.396713:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.396717:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176008.396722:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176008.396727:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176008.396736:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176008.396744:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176008.396756:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235438720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176008.396770:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176008.396776:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176008.396783:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235438720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176008.396811:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176008.396816:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176008.396846:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176008.396848:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176008.396888:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176008.396892:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176008.396896:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10682 00000400:00000200:3.0:1786176008.396901:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176008.396904:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176008.396907:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176008.396911:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.396914:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176008.396918:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.396921:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176008.396925:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176008.396927:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176008.396930:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176008.396932:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176008.396935:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176008.396937:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176008.396943:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.396951:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176008.396955:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.396961:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176008.396966:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176008.396974:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176008.396986:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176008.397007:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176008.397021:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176008.397035:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.397047:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176008.397058:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176008.397069:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.397074:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176008.397085:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786176008.397092:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176008.397101:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176008.397105:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.397108:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176008.397111:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176008.397116:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176008.397129:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176008.397142:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176008.397151:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235438848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176008.397165:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176008.397172:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176008.397180:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235438848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176008.397202:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176008.397208:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176008.397219:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176008.397225:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176008.397229:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176008.397236:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176008.397258:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10682 00000400:00000200:3.0:1786176008.397270:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176008.397278:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176008.397282:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176008.397289:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.397296:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176008.397301:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.397308:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176008.397317:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176008.397320:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176008.397325:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176008.397330:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176008.397335:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176008.397344:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176008.397388:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.397392:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176008.397395:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.397398:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176008.397401:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176008.397406:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176008.397411:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176008.397416:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176008.397423:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176008.397431:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.397436:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176008.397440:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176008.397448:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.397453:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176008.397460:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176008.397467:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176008.397474:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176008.397477:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.397480:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176008.397482:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176008.397488:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176008.397497:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176008.397507:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176008.397522:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235438976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1786176008.397523:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176008.397534:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000100:00000001:3.0:1786176008.397544:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176008.397551:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176008.397563:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235438976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176008.397586:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176008.397592:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176008.397602:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176008.397607:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176008.397610:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176008.397617:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176008.397624:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10682 00000400:00000200:3.0:1786176008.397637:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176008.397646:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176008.397651:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176008.397658:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.397662:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176008.397669:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.397694:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176008.397703:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176008.397707:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176008.397712:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176008.397718:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176008.397724:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176008.397733:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176008.397743:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.397754:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176008.397760:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.397767:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176008.397774:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176008.397782:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176008.397787:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176008.397796:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176008.397800:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176008.397803:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.397811:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176008.397816:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176008.397843:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.397846:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176008.397853:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786176008.397862:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176008.397890:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176008.397892:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.397899:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176008.397902:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176008.397908:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176008.397920:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176008.397929:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176008.397945:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235439104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176008.397960:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176008.397966:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176008.397978:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235439104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176008.398022:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176008.398034:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176008.398049:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176008.398058:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176008.398064:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176008.398074:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176008.398084:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10682 00000400:00000200:3.0:1786176008.398095:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176008.398104:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176008.398111:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176008.398120:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.398127:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176008.398133:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.398141:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176008.398165:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176008.398170:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176008.398178:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176008.398182:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176008.398194:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176008.398203:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176008.398209:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.398217:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176008.398221:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176008.398227:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176008.398234:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176008.398243:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176008.398251:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176008.398257:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176008.398266:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176008.398273:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.398281:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176008.398287:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176008.398297:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.398305:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176008.398315:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176008.398322:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176008.398329:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176008.398337:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176008.398346:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176008.398350:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176008.398356:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176008.398386:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176008.398395:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176008.398439:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235439232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176008.398480:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176008.398486:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176008.398495:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235439232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176008.398514:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176008.398524:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10692) 00000800:00000200:1.1:1786176008.398708:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176008.398711:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786176008.399162:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176008.399168:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786176008.399566:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176008.399569:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786176008.399977:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176008.399980:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000020:00000001:3.0:1786176009.292741:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176009.292747:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176009.292752:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176010.316542:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176010.316545:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176010.316548:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176011.340647:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176011.340654:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176011.340659:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176012.364577:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176012.364582:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176012.364586:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176013.388535:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176013.388539:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176013.388542:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176013.517091:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176013.517097:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176013.517853:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176013.517856:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176013.518127:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176013.518129:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176013.518362:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176013.518364:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176013.518753:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176013.518755:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000020:00000001:3.0:1786176014.412648:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176014.412654:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176014.412659:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176015.436691:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176015.436699:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176015.436707:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176016.460522:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176016.460525:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176016.460527:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176017.484614:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176017.484620:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176017.484624:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176018.508568:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176018.508572:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176018.508576:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176018.637163:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176018.637169:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176018.638044:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176018.638047:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176018.638355:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176018.638377:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176018.638712:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176018.638714:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176018.639136:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176018.639139:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000020:00000001:3.0:1786176019.532602:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176019.532608:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176019.532612:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176020.556561:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176020.556567:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176020.556571:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176021.580589:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176021.580595:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176021.580599:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176022.604657:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176022.604664:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176022.604669:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176023.628757:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176023.628764:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176023.628770:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176023.757426:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176023.757468:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176023.758358:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176023.758361:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176023.758659:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176023.758662:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176023.759119:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176023.759121:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176023.759484:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176023.759486:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1786176024.269291:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176024.269309:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176024.269312:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176024.269319:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176024.269326:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176024.269332:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176024.269335:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176024.269338:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176024.269342:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176024.652584:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176024.652587:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176024.652591:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176025.292494:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176025.292504:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176025.292507:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176025.292514:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176025.292520:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176025.292524:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176025.292528:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176025.292532:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176025.292537:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176025.676540:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176025.676545:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176025.676548:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176026.700577:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176026.700581:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176026.700584:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176027.724585:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176027.724590:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176027.724594:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176028.748525:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176028.748528:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176028.748531:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176028.876982:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176028.876986:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176028.877606:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176028.877608:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176028.877882:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176028.877884:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176028.878143:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176028.878146:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176028.878470:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176028.878474:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000020:00000001:3.0:1786176029.772562:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176029.772568:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176029.772572:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176030.796590:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176030.796598:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176030.796637:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176031.820538:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176031.820542:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176031.820546:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176032.844539:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176032.844542:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176032.844545:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176033.868640:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176033.868644:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176033.868647:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176033.997244:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176033.997253:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786176033.998096:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176033.998098:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786176033.998421:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176033.998425:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786176033.998736:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176033.998738:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786176033.999012:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176033.999015:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000020:00000001:3.0:1786176034.892553:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176034.892558:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176034.892575:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176035.916679:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176035.916683:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176035.916687:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176036.940738:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176036.940746:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176036.940752:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176037.964723:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176037.964731:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176037.964737:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176038.988555:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176038.988559:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176038.988563:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176039.117073:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176039.117613:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176039.117782:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176039.117961:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176039.118127:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:3.0:1786176040.012539:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176040.012554:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176040.012560:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176041.036562:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176041.036565:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176041.036568:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176041.676587:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176041.676593:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176041.676596:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176041.676599:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176041.676602:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176041.676606:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176041.676608:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176041.676610:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176041.676612:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176042.060590:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176042.060594:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176042.060596:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176042.700468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176042.700475:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176042.700477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176042.700482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176042.700485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176042.700508:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176042.700511:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176042.700514:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176042.700516:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176043.084611:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176043.084616:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176043.084621:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176043.724446:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176043.724453:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176043.724456:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176043.724462:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176043.724467:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176043.724470:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176043.724474:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176043.724489:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176043.724493:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176044.108680:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176044.108683:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176044.108686:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176044.237338:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176044.237363:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176044.238240:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176044.238243:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176044.238641:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176044.238644:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176044.239017:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176044.239020:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176044.239505:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176044.239509:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000020:00000001:3.0:1786176045.132670:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176045.132676:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176045.132680:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176046.156844:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176046.156851:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176046.156856:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176047.180545:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176047.180549:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176047.180552:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176048.204578:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176048.204582:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176048.204584:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176049.228783:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176049.228791:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176049.228797:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176049.357504:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176049.357531:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176049.358930:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176049.358934:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176049.359293:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176049.359296:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176049.359803:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176049.359806:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176049.360263:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176049.360267:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000020:00000001:3.0:1786176050.252659:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176050.252667:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176050.252674:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176051.276629:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176051.276636:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176051.276640:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176051.916809:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176051.916820:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176051.916823:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176051.916832:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176051.916839:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176051.916846:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176051.916851:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176051.916855:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176051.916860:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176052.300782:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176052.300792:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176052.300800:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176052.940532:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176052.940544:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176052.940551:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176052.940559:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176052.940570:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176052.940576:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176052.940580:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176052.940585:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176052.940591:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176053.324679:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176053.324688:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176053.324696:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176053.964518:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176053.964531:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176053.964536:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176053.964547:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176053.964554:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176053.964560:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176053.964565:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176053.964570:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176053.964576:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176054.348687:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176054.348694:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176054.348700:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176054.477516:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176054.477528:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176054.478698:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176054.478703:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176054.479076:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176054.479082:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176054.479598:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176054.479602:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176054.479952:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176054.479955:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1786176054.988467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176054.988476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176054.988479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176054.988483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176054.988488:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176054.988491:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176054.988494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176054.988498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176054.988501:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176055.372592:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176055.372596:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176055.372600:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176056.396746:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176056.396753:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176056.396758:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176057.420628:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176057.420637:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176057.420642:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176058.060573:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176058.060583:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176058.060586:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176058.060592:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176058.060597:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176058.060602:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176058.060605:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176058.060609:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176058.060613:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176058.444561:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176058.444565:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176058.444568:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176059.084461:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176059.084470:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176059.084472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176059.084477:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176059.084512:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176059.084518:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176059.084522:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176059.084526:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176059.084530:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176059.468540:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176059.468546:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176059.468551:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176059.597125:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176059.597132:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176059.597896:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176059.597898:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176059.598123:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176059.598125:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176059.598347:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176059.598350:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176059.598628:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176059.598631:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1786176060.108477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176060.108484:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176060.108487:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176060.108491:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176060.108495:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176060.108498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176060.108500:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176060.108502:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176060.108505:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176060.492499:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176060.492505:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176060.492509:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176061.132455:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176061.132465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176061.132467:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176061.132473:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176061.132477:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176061.132480:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176061.132483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176061.132487:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176061.132490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176061.516578:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176061.516586:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176061.516594:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176062.156485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176062.156493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176062.156496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176062.156502:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176062.156507:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176062.156511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176062.156514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176062.156518:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176062.156524:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176062.540649:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176062.540655:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176062.540660:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176063.564570:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176063.564576:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176063.564580:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176064.588611:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176064.588617:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176064.588622:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176064.717572:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176064.717580:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176064.718622:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176064.718624:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176064.719028:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176064.719032:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176064.719514:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176064.719517:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176064.719798:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176064.719801:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000020:00000001:3.0:1786176065.612725:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176065.612731:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176065.612736:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176066.636752:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176066.636759:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176066.636764:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176067.276561:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176067.276572:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176067.276575:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176067.276581:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176067.276587:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176067.276591:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176067.276594:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176067.276599:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176067.276603:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176067.660655:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176067.660662:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176067.660667:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176068.300515:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176068.300526:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176068.300529:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176068.300537:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176068.300544:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176068.300549:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176068.300553:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176068.300557:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176068.300561:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176068.684721:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176068.684730:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176068.684736:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176069.324485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176069.324496:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176069.324499:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176069.324506:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176069.324513:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176069.324519:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176069.324523:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176069.324527:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176069.324531:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176069.708706:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176069.708714:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176069.708719:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176069.837147:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176069.837151:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176069.837985:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176069.837986:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176069.838177:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176069.838179:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176069.838458:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176069.838460:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176069.838750:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176069.838752:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1786176070.348489:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176070.348498:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176070.348501:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176070.348506:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176070.348510:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176070.348514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176070.348517:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176070.348520:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176070.348524:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176070.732648:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176070.732654:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176070.732658:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176071.372472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176071.372479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176071.372482:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176071.372486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176071.372492:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176071.372496:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176071.372498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176071.372500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176071.372503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176071.756614:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176071.756618:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176071.756621:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176072.396471:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176072.396482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176072.396486:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176072.396494:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176072.396500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176072.396506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176072.396510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176072.396516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176072.396522:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176072.780595:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176072.780599:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176072.780602:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176073.420447:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176073.420455:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176073.420457:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176073.420462:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176073.420466:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176073.420469:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176073.420472:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176073.420474:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176073.420478:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176073.804671:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176073.804680:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176073.804688:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176074.444508:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176074.444520:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176074.444523:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176074.444529:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176074.444533:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176074.444537:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176074.444541:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176074.444544:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176074.444548:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176074.828612:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176074.828622:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176074.828630:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176074.957643:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176074.957653:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176074.958775:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176074.958778:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176074.959231:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176074.959235:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176074.959726:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176074.959729:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176074.960147:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176074.960151:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1786176075.468523:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176075.468537:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176075.468540:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176075.468550:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176075.468555:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176075.468562:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176075.468566:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176075.468570:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176075.468574:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176075.852734:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176075.852741:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176075.852745:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176076.492498:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176076.492510:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176076.492513:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176076.492521:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176076.492526:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176076.492530:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176076.492533:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176076.492536:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176076.492539:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176076.876505:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176076.876508:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176076.876510:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176077.900651:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176077.900657:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176077.900662:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176078.924536:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176078.924540:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176078.924543:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176079.948621:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176079.948627:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176079.948633:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176080.077521:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176080.077530:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176080.078681:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176080.078684:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176080.079117:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176080.079121:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176080.079567:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176080.079570:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176080.080074:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176080.080077:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1786176080.588524:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176080.588562:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176080.588566:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176080.588574:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176080.588580:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176080.588584:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176080.588588:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176080.588591:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176080.588595:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176080.972565:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176080.972572:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176080.972578:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176081.612476:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176081.612488:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176081.612491:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176081.612500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176081.612506:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176081.612511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176081.612515:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176081.612519:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176081.612523:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176081.996567:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176081.996577:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176081.996586:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176082.636490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176082.636515:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176082.636519:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176082.636529:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176082.636538:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176082.636549:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176082.636554:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176082.636560:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176082.636566:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176083.020630:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176083.020639:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176083.020646:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176083.660436:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176083.660446:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176083.660449:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176083.660456:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176083.660460:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176083.660464:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176083.660468:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176083.660470:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176083.660473:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176084.044572:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176084.044579:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176084.044584:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176084.684448:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176084.684456:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176084.684459:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176084.684465:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176084.684470:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176084.684477:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176084.684481:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176084.684486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176084.684492:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176085.068639:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176085.068645:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176085.068649:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176085.197061:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176085.197068:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176085.197691:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176085.197693:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176085.197884:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176085.197885:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176085.198121:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176085.198123:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176085.198339:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176085.198341:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1786176085.708493:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176085.708505:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176085.708509:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176085.708516:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176085.708522:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176085.708527:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176085.708531:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176085.708535:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176085.708540:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176086.092686:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176086.092696:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176086.092702:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176087.116710:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176087.116719:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176087.116726:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176088.140547:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176088.140551:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176088.140555:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176089.164637:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176089.164641:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176089.164644:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176090.188555:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176090.188559:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176090.188563:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176090.316542:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176090.316552:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176090.316554:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176090.316558:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176090.316562:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176090.316568:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10764 00000400:00000200:3.0:1786176090.316575:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176090.316581:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176090.316583:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176090.316586:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316588:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176090.316590:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316592:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176090.316596:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176090.316597:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176090.316600:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176090.316602:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176090.316604:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176090.316605:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176090.316607:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316609:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176090.316610:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316612:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176090.316614:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176090.316617:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316618:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176090.316620:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176090.316623:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176090.316625:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.316626:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176090.316628:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176090.316636:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176090.316638:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176090.316641:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176090.316643:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.316645:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176090.316647:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176090.316651:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176090.316657:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176090.316662:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176090.316670:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235448960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176090.316677:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176090.316682:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176090.316685:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235448960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176090.316702:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176090.316705:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176090.316708:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176090.316709:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176090.316710:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176090.316712:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176090.316713:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10764 00000400:00000200:3.0:1786176090.316716:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176090.316718:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176090.316721:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176090.316724:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316727:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176090.316730:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316733:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176090.316736:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176090.316737:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176090.316740:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176090.316742:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176090.316745:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176090.316750:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176090.316757:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316762:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176090.316769:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316776:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176090.316781:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176090.316788:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316792:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176090.316796:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176090.316802:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176090.316806:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.316808:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176090.316812:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176090.316817:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.316819:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176090.316823:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176090.316827:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176090.316832:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176090.316835:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.316837:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176090.316838:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176090.316840:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176090.316847:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176090.316851:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176090.316858:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235449088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176090.316865:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176090.316869:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176090.316875:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235449088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176090.316884:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176090.316888:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176090.316894:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176090.316897:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176090.316900:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176090.316904:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176090.316909:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10764 00000400:00000200:3.0:1786176090.316916:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176090.316920:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176090.316924:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176090.316929:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316931:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176090.316935:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316938:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176090.316940:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176090.316941:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176090.316942:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176090.316945:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176090.316947:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176090.316952:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176090.316954:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316956:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176090.316957:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316962:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176090.316966:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176090.316968:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176090.316972:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176090.316976:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176090.316986:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176090.316990:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.316992:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176090.316996:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176090.316999:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.317001:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176090.317006:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176090.317010:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176090.317014:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176090.317017:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.317020:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176090.317022:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176090.317026:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176090.317046:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176090.317049:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176090.317055:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235449216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176090.317077:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176090.317080:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176090.317087:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235449216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176090.317097:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176090.317101:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176090.317107:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176090.317110:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176090.317113:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176090.317118:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1786176090.317119:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786176090.317123:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10764 00000800:00000200:1.1:1786176090.317124:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1786176090.317129:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176090.317137:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176090.317139:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176090.317143:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.317145:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176090.317151:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.317154:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176090.317157:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176090.317160:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176090.317163:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176090.317167:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176090.317170:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176090.317173:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176090.317178:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.317182:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176090.317187:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.317191:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176090.317196:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176090.317200:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176090.317204:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176090.317206:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176090.317210:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176090.317215:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.317219:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176090.317222:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176090.317225:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.317228:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176090.317232:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176090.317237:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176090.317242:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176090.317246:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.317248:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176090.317251:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176090.317253:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176090.317257:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176090.317260:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176090.317266:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235449344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176090.317282:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176090.317286:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176090.317291:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235449344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176090.317303:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176090.317305:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176090.317312:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176090.317315:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176090.317317:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176090.317322:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176090.317325:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10764 00000400:00000200:3.0:1786176090.317330:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176090.317336:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176090.317340:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176090.317345:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.317348:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176090.317354:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.317395:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176090.317401:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176090.317403:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176090.317406:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176090.317410:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176090.317415:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176090.317420:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176090.317424:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.317427:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176090.317431:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176090.317434:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176090.317437:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176090.317444:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176090.317450:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176090.317454:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176090.317458:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176090.317463:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.317465:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176090.317467:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176090.317470:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.317472:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176090.317478:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176090.317482:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176090.317486:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176090.317488:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176090.317493:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176090.317495:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176090.317497:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176090.317504:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176090.317510:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176090.317518:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235449472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176090.317534:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176090.317538:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176090.317544:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235449472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176090.317556:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176090.317562:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10774) 00000800:00000200:1.1:1786176090.317714:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176090.317716:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176090.317900:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176090.317902:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176090.318118:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176090.318120:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176090.318297:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176090.318299:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000020:00000001:3.0:1786176091.212649:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176091.212654:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176091.212658:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176092.236561:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176092.236567:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176092.236572:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176094.284549:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176094.284555:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176094.284560:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176095.436969:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176095.436988:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176095.437568:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176095.437570:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176095.437795:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176095.437797:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176095.437993:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176095.437995:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176095.438251:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176095.438252:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000020:00000001:3.0:1786176096.332592:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176096.332597:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176096.332599:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176098.380546:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176098.380550:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176098.380554:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176100.428549:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176100.428555:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176100.428559:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176100.556552:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176100.556568:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176100.556572:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176100.556578:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176100.556584:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176100.556595:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10775 00000400:00000200:3.0:1786176100.556607:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176100.556623:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176100.556627:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176100.556630:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.556634:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176100.556636:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.556639:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176100.556642:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176100.556645:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176100.556647:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176100.556651:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176100.556653:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176100.556655:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176100.556658:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.556660:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176100.556662:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.556664:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176100.556666:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176100.556669:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176100.556671:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176100.556674:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176100.556679:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176100.556683:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.556686:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176100.556689:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176100.556702:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176100.556707:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176100.556712:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176100.556715:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.556719:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176100.556724:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176100.556730:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176100.556740:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176100.556747:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176100.556758:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235450240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176100.556768:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176100.556772:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176100.556777:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235450240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176100.556801:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176100.556804:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176100.556807:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176100.556808:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176100.556810:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176100.556812:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176100.556815:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10775 00000400:00000200:3.0:1786176100.556819:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176100.556822:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176100.556824:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176100.556837:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.556839:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176100.556841:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.556849:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176100.556851:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176100.556853:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176100.556855:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176100.556858:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176100.556860:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176100.556862:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176100.556865:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.556869:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176100.556874:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.556881:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176100.556890:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176100.556902:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176100.556916:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176100.556942:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176100.556952:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176100.556960:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.556964:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176100.556981:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176100.556994:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.557004:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176100.557014:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176100.557021:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176100.557029:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176100.557034:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.557039:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176100.557044:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176100.557046:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176100.557061:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176100.557068:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176100.557080:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235450368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176100.557097:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176100.557100:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176100.557111:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235450368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176100.557138:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176100.557149:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176100.557167:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176100.557172:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176100.557173:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176100.557175:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176100.557197:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10775 00000400:00000200:3.0:1786176100.557219:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176100.557225:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176100.557235:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176100.557256:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557265:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176100.557271:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557275:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176100.557279:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176100.557282:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176100.557289:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176100.557291:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176100.557294:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176100.557304:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176100.557317:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557323:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176100.557331:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557338:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176100.557400:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176100.557406:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557413:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176100.557415:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176100.557418:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176100.557424:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.557426:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176100.557427:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176100.557430:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.557432:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176100.557435:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176100.557439:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176100.557442:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176100.557443:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.557445:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176100.557448:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176100.557452:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176100.557456:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176100.557460:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176100.557467:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235450496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176100.557482:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176100.557484:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176100.557490:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235450496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1786176100.557499:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176100.557512:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000100:00000001:3.0:1786176100.557520:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176100.557529:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176100.557539:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176100.557545:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176100.557549:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176100.557556:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176100.557562:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10775 00000400:00000200:3.0:1786176100.557569:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176100.557580:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176100.557582:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176100.557586:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557593:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176100.557596:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557602:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176100.557606:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176100.557607:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176100.557609:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176100.557612:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176100.557615:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176100.557620:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176100.557629:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557634:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176100.557638:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557641:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176100.557649:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176100.557656:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557661:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176100.557668:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176100.557673:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176100.557680:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.557685:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176100.557691:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176100.557703:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.557711:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176100.557719:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786176100.557725:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176100.557732:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176100.557736:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.557741:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176100.557750:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176100.557754:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176100.557766:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176100.557773:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176100.557781:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235450624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176100.557805:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176100.557810:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176100.557820:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235450624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176100.557853:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176100.557857:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176100.557862:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176100.557869:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176100.557877:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176100.557890:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176100.557902:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10775 00000400:00000200:3.0:1786176100.557918:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176100.557943:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176100.557946:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176100.557955:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557964:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176100.557971:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.557986:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176100.557994:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176100.558003:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176100.558009:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176100.558014:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176100.558022:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176100.558029:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176100.558039:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.558044:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176100.558053:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176100.558059:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176100.558067:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176100.558074:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176100.558085:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176100.558093:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176100.558100:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176100.558114:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.558118:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176100.558121:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176100.558134:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.558144:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176100.558153:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176100.558171:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176100.558175:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176100.558185:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176100.558202:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176100.558210:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176100.558217:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176100.558233:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176100.558244:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176100.558260:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235450752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176100.558396:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176100.558402:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176100.558412:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235450752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176100.558423:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176100.558430:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10785) 00000800:00000200:1.1:1786176100.558571:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176100.558573:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786176100.558837:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176100.558839:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786176100.559149:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176100.559151:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786176100.559436:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176100.559439:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000020:00000001:3.0:1786176102.476611:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176102.476616:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176102.476619:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176104.524581:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176104.524585:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176104.524589:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176105.676452:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176105.676466:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176105.676469:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176105.676486:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176105.676503:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176105.676513:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10780 00000400:00000200:3.0:1786176105.676524:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176105.676531:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176105.676535:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176105.676539:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.676544:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176105.676547:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.676550:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176105.676565:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176105.676573:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176105.676577:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176105.676581:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176105.676584:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176105.676588:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176105.676592:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.676595:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176105.676597:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.676600:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176105.676604:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176105.676607:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176105.676610:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176105.676614:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176105.676617:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176105.676630:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.676633:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176105.676636:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176105.676647:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176105.676652:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176105.676656:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176105.676658:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.676662:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176105.676665:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176105.676671:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176105.676680:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176105.676688:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176105.676699:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235450880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176105.676712:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176105.676718:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176105.676724:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235450880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176105.676753:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176105.676759:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176105.676765:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176105.676769:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176105.676772:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176105.676787:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176105.676792:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10780 00000400:00000200:3.0:1786176105.676800:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176105.676806:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176105.676810:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176105.676815:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.676820:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176105.676823:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.676827:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176105.676832:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176105.676834:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176105.676837:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176105.676841:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176105.676855:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176105.676859:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176105.676865:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.676868:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176105.676872:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.676876:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176105.676907:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176105.676923:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176105.676928:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176105.676931:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176105.676939:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176105.676943:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.676947:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176105.676953:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176105.676958:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.676962:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176105.676969:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786176105.676973:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176105.676978:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176105.676980:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.676983:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176105.676986:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176105.676989:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176105.676996:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176105.677015:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176105.677036:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235451008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176105.677051:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176105.677056:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176105.677064:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235451008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176105.677082:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176105.677087:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176105.677094:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176105.677096:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176105.677100:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176105.677104:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176105.677108:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10780 00000400:00000200:3.0:1786176105.677115:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176105.677119:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176105.677124:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176105.677129:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677133:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176105.677137:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677141:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176105.677145:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176105.677148:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176105.677151:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176105.677155:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176105.677159:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176105.677163:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176105.677169:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677182:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176105.677185:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677188:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176105.677193:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176105.677198:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677203:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176105.677206:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176105.677212:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176105.677216:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.677230:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:1.1:1786176105.677244:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000008:00100000:3.0:1786176105.677245:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176105.677249:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.677253:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176105.677259:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786176105.677263:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176105.677268:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176105.677271:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.677274:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176105.677276:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176105.677280:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176105.677287:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176105.677293:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176105.677323:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235451136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176105.677337:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176105.677342:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176105.677350:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235451136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176105.677414:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176105.677418:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176105.677426:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176105.677430:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176105.677433:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176105.677439:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176105.677444:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10780 00000400:00000200:3.0:1786176105.677451:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176105.677458:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176105.677463:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176105.677470:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677484:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176105.677488:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677493:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176105.677497:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176105.677500:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176105.677503:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176105.677507:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176105.677512:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176105.677517:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176105.677523:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677527:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176105.677532:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677536:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176105.677540:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176105.677554:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677559:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176105.677563:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176105.677568:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176105.677572:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.677576:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176105.677581:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176105.677586:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.677590:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176105.677598:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176105.677603:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176105.677620:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176105.677623:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.677627:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176105.677630:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176105.677634:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176105.677641:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176105.677647:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176105.677655:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235451264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176105.677670:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176105.677674:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176105.677682:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235451264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176105.677699:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176105.677705:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176105.677711:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176105.677715:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176105.677718:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176105.677722:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176105.677726:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10780 00000400:00000200:3.0:1786176105.677733:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176105.677738:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000200:1.1:1786176105.677739:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786176105.677743:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176105.677748:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677752:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176105.677757:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677761:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176105.677765:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176105.677767:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176105.677771:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176105.677775:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176105.677779:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176105.677784:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176105.677790:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677793:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176105.677797:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677802:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176105.677806:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176105.677811:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176105.677816:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176105.677819:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176105.677824:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176105.677828:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.677832:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176105.677836:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176105.677840:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.677844:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176105.677849:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786176105.677854:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176105.677859:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176105.677861:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176105.677865:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176105.677868:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176105.677871:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176105.677877:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176105.677928:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176105.677946:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235451392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176105.677961:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176105.677965:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176105.677971:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235451392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176105.677985:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176105.677993:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10790) 00000800:00000200:1.1:1786176105.678086:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176105.678405:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176105.678650:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:3.0:1786176106.572653:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176106.572657:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176106.572660:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176110.796463:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176110.796479:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176110.796483:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176110.796488:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176110.796496:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176110.796508:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10785 00000400:00000200:3.0:1786176110.796520:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176110.796530:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176110.796551:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176110.796555:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.796560:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176110.796563:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.796565:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176110.796570:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176110.796573:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176110.796576:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176110.796581:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176110.796584:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176110.796587:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176110.796592:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.796594:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176110.796597:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.796599:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176110.796603:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176110.796606:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176110.796609:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176110.796613:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176110.796617:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176110.796621:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.796624:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176110.796627:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176110.796651:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176110.796656:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176110.796661:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176110.796664:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.796669:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176110.796673:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176110.796680:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176110.796690:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176110.796700:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176110.796714:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235451520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176110.796743:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176110.796749:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176110.796755:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235451520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176110.796786:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176110.796790:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176110.796794:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176110.796795:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176110.796798:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176110.796801:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176110.796804:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10785 00000400:00000200:3.0:1786176110.796808:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176110.796811:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176110.796814:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176110.796817:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.796820:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176110.796823:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.796827:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176110.796845:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176110.796846:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176110.796849:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176110.796852:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176110.796857:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176110.796860:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176110.796865:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.796867:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176110.796871:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.796874:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176110.796876:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176110.796879:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176110.796883:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176110.796887:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176110.796899:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176110.796906:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.796913:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176110.796920:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176110.796931:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.796943:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176110.796954:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176110.796969:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176110.796976:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176110.796981:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.796986:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176110.796990:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176110.796994:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176110.797002:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176110.797010:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176110.797023:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235451648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176110.797041:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176110.797045:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176110.797052:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235451648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176110.797075:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176110.797083:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176110.797088:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176110.797090:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176110.797092:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176110.797095:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176110.797103:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10785 00000400:00000200:3.0:1786176110.797117:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176110.797143:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176110.797151:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176110.797165:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.797174:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176110.797179:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.797186:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176110.797194:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176110.797199:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176110.797206:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176110.797214:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176110.797223:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176110.797233:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176110.797244:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.797252:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176110.797260:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.797267:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176110.797275:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176110.797280:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176110.797289:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176110.797295:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176110.797300:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176110.797303:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.797306:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176110.797309:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176110.797316:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.797323:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176110.797328:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176110.797336:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176110.797345:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176110.797350:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.797394:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176110.797396:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176110.797399:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176110.797405:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176110.797409:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176110.797418:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235451776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176110.797427:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1786176110.797504:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176110.797508:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786176110.797521:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176110.797533:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235451776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176110.797558:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176110.797566:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176110.797574:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176110.797577:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176110.797583:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176110.797590:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176110.797598:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10785 00000400:00000200:3.0:1786176110.797613:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176110.797623:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176110.797632:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176110.797644:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.797650:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176110.797657:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.797665:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176110.797676:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176110.797680:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176110.797686:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176110.797691:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176110.797694:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176110.797698:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176110.797710:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.797719:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176110.797726:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.797731:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176110.797736:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000800:00000200:1.1:1786176110.798141:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176110.798144:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00080000:3.0:1786176110.798217:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176110.798228:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176110.798235:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176110.798242:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176110.798252:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.798260:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176110.798267:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176110.798275:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.798281:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176110.798291:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176110.798298:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176110.798307:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176110.798312:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.798317:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176110.798323:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176110.798330:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176110.798343:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176110.798354:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176110.798431:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235451904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176110.798463:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176110.798469:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176110.798481:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235451904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176110.798513:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176110.798520:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176110.798532:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176110.798537:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176110.798541:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176110.798550:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176110.798561:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10785 00000400:00000200:3.0:1786176110.798571:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176110.798581:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176110.798585:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176110.798596:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.798603:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176110.798613:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.798620:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176110.798627:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176110.798631:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176110.798636:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176110.798640:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176110.798647:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176110.798653:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176110.798665:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.798673:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176110.798682:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176110.798688:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176110.798695:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176110.798701:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176110.798706:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176110.798711:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176110.798715:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176110.798719:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.798726:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176110.798732:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176110.798740:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.798743:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176110.798753:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176110.798761:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176110.798766:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176110.798769:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176110.798776:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176110.798780:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176110.798785:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176110.798794:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176110.798800:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176110.798810:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235452032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176110.798822:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176110.798826:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176110.798845:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235452032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176110.798863:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176110.798872:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10795) 00000800:00000200:1.1:1786176110.798928:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176110.798931:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176110.799183:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176110.799184:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176110.799400:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176110.799402:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176115.917109:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176115.917113:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176115.917587:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176115.917589:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176115.917771:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176115.917772:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176115.917969:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176115.917971:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176115.918181:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176115.918183:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00080000:3.0:1786176121.036488:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176121.036504:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176121.036507:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176121.036511:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176121.036516:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176121.036523:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10795 00000400:00000200:3.0:1786176121.036532:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176121.036539:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176121.036543:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176121.036547:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036551:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176121.036554:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036556:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176121.036561:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176121.036563:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176121.036566:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176121.036569:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176121.036572:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176121.036575:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176121.036578:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036579:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176121.036582:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036584:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176121.036587:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176121.036590:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036593:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176121.036596:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176121.036600:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176121.036603:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.036605:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176121.036607:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176121.036617:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176121.036621:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176121.036624:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176121.036626:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.036629:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176121.036631:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176121.036635:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176121.036643:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176121.036650:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176121.036661:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235452800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176121.036672:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176121.036677:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176121.036682:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235452800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176121.036699:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176121.036702:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176121.036705:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176121.036706:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176121.036707:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176121.036709:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176121.036711:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10795 00000400:00000200:3.0:1786176121.036715:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176121.036718:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176121.036720:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176121.036722:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036724:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176121.036726:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036743:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176121.036749:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176121.036750:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176121.036753:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176121.036755:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176121.036759:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176121.036764:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176121.036767:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036768:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176121.036770:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036774:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176121.036777:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176121.036782:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036786:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176121.036789:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176121.036794:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176121.036798:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.036804:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176121.036809:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176121.036813:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.036817:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176121.036822:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176121.036824:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176121.036830:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176121.036833:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.036836:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176121.036837:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176121.036840:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176121.036846:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176121.036851:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176121.036856:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235452928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176121.036863:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176121.036868:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176121.036874:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235452928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176121.036886:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176121.036892:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176121.036899:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176121.036902:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176121.036905:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176121.036908:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176121.036912:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10795 00000400:00000200:3.0:1786176121.036920:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176121.036924:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176121.036925:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176121.036931:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036934:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176121.036936:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036939:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176121.036944:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176121.036945:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176121.036948:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176121.036952:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176121.036955:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176121.036960:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176121.036966:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036969:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176121.036972:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036975:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176121.036978:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176121.036983:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176121.036987:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176121.036991:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176121.036994:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176121.036997:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.037000:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176121.037002:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176121.037004:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.037005:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176121.037008:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176121.037011:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176121.037015:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176121.037017:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.037020:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176121.037023:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176121.037024:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176121.037027:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176121.037028:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176121.037035:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235453056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176121.037048:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176121.037050:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176121.037056:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235453056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176121.037069:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176121.037074:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.1:1786176121.037079:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786176121.037080:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176121.037084:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.1:1786176121.037085:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00080000:3.0:1786176121.037086:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176121.037092:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176121.037097:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10795 00000400:00000200:3.0:1786176121.037103:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176121.037107:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176121.037109:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176121.037116:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.037120:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176121.037125:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.037127:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176121.037129:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176121.037132:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176121.037133:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176121.037135:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176121.037136:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176121.037138:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176121.037143:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.037145:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176121.037149:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.037152:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176121.037155:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176121.037161:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176121.037166:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176121.037170:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176121.037174:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176121.037177:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.037181:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176121.037185:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176121.037189:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.037192:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176121.037198:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176121.037202:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176121.037205:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176121.037207:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.037213:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176121.037214:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176121.037216:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176121.037221:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176121.037225:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176121.037231:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235453184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176121.037237:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176121.037241:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176121.037244:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235453184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176121.037251:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176121.037253:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176121.037258:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176121.037261:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176121.037263:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176121.037268:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176121.037272:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10795 00000400:00000200:3.0:1786176121.037278:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176121.037284:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176121.037288:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176121.037294:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.037299:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176121.037302:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.037308:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176121.037310:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176121.037311:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176121.037315:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176121.037317:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176121.037320:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176121.037322:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176121.037329:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.037331:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176121.037336:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176121.037339:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176121.037342:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176121.037345:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176121.037348:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176121.037350:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176121.037353:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176121.037382:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.037383:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176121.037385:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176121.037386:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.037390:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176121.037395:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786176121.037399:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176121.037402:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176121.037404:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176121.037408:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176121.037410:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176121.037412:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176121.037418:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176121.037424:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176121.037432:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235453312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176121.037463:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176121.037467:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176121.037475:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235453312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176121.037490:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176121.037494:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10805) 00000800:00000200:2.1:1786176121.037689:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176121.037690:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176121.037903:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176121.037905:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176121.038120:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176121.038122:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176121.038297:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176121.038301:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.0:1786176121.484466:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:3.0:1786176121.484477:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:3.0:1786176121.484483:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:1.1:1786176126.156930:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176126.156934:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176126.157447:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176126.157449:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176126.157652:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176126.157654:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176126.157873:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176126.157875:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176126.158067:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176126.158069:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786176131.277210:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176131.277218:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176131.277896:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176131.277899:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176131.278199:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176131.278201:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176131.278438:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176131.278440:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176131.278627:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176131.278641:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00080000:3.0:1786176136.396464:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176136.396476:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176136.396478:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176136.396482:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176136.396486:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176136.396493:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10810 00000400:00000200:3.0:1786176136.396502:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176136.396509:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176136.396512:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176136.396515:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.396519:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176136.396521:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.396524:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176136.396528:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176136.396530:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176136.396532:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176136.396535:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176136.396537:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176136.396540:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176136.396543:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.396545:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176136.396547:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.396549:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176136.396552:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176136.396555:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176136.396557:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176136.396559:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176136.396562:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176136.396565:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.396567:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176136.396570:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176136.396578:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176136.396602:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176136.396606:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176136.396608:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.396611:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176136.396614:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176136.396618:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176136.396642:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176136.396647:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176136.396655:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235454720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176136.396664:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176136.396668:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176136.396671:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235454720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176136.396688:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176136.396690:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176136.396693:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176136.396694:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176136.396695:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176136.396697:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176136.396699:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10810 00000400:00000200:3.0:1786176136.396702:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176136.396705:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176136.396707:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176136.396710:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.396712:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176136.396715:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.396716:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176136.396718:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176136.396720:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176136.396724:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176136.396726:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176136.396730:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176136.396735:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176136.396739:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.396742:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176136.396746:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.396750:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176136.396754:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176136.396758:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176136.396763:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176136.396766:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176136.396773:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176136.396776:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.396778:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176136.396784:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176136.396790:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.396792:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176136.396799:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786176136.396803:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176136.396809:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176136.396813:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.396818:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176136.396822:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176136.396827:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176136.396837:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176136.396844:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176136.396854:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235454848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176136.396873:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176136.396878:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176136.396889:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235454848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176136.396907:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176136.396911:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176136.396917:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176136.396919:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176136.396920:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176136.396925:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176136.396931:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10810 00000400:00000200:3.0:1786176136.396977:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176136.396985:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176136.396990:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176136.396994:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.396998:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176136.397000:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397001:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176136.397005:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176136.397007:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176136.397011:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176136.397014:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176136.397017:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176136.397021:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176136.397026:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397029:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1786176136.397030:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786176136.397033:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397038:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:2.1:1786176136.397039:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000020:00000040:3.0:1786176136.397040:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176136.397045:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397048:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176136.397050:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176136.397054:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176136.397057:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.397060:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176136.397063:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176136.397067:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.397068:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176136.397071:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786176136.397072:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176136.397074:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176136.397076:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.397081:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176136.397084:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176136.397087:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176136.397090:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176136.397097:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176136.397104:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235454976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176136.397131:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176136.397135:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176136.397146:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235454976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176136.397175:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176136.397185:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176136.397189:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176136.397192:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176136.397193:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176136.397196:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176136.397200:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10810 00000400:00000200:3.0:1786176136.397203:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176136.397209:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176136.397210:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176136.397213:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397218:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176136.397220:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397224:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176136.397229:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176136.397231:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176136.397234:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176136.397237:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176136.397241:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176136.397245:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176136.397250:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397254:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176136.397258:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397262:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176136.397267:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176136.397271:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397292:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176136.397298:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176136.397302:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176136.397308:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.397310:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176136.397316:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176136.397321:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.397327:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176136.397335:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786176136.397358:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176136.397366:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176136.397371:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.397377:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176136.397381:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176136.397424:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176136.397436:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176136.397446:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176136.397464:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235455104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176136.397507:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176136.397516:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176136.397532:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235455104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176136.397554:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176136.397561:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176136.397569:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176136.397573:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176136.397577:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176136.397599:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176136.397606:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10810 00000400:00000200:3.0:1786176136.397615:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176136.397623:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176136.397646:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176136.397651:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397653:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176136.397657:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397661:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176136.397664:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176136.397667:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176136.397671:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176136.397675:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176136.397680:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176136.397682:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176136.397687:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397693:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176136.397697:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397702:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176136.397707:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176136.397709:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176136.397710:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176136.397712:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176136.397716:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176136.397720:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.397723:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176136.397726:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176136.397731:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.397735:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176136.397741:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176136.397744:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176136.397747:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176136.397750:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176136.397754:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176136.397756:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176136.397761:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176136.397767:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176136.397771:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176136.397779:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235455232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176136.397798:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176136.397802:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176136.397807:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235455232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176136.397823:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176136.397830:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10820) 00000800:00000200:2.1:1786176136.397902:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176136.397905:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176136.398138:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176136.398140:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176136.398421:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176136.398423:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176136.398810:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176136.398812:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000100:00080000:3.0:1786176141.516482:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176141.516496:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176141.516500:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176141.516504:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176141.516511:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176141.516520:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10815 00000400:00000200:3.0:1786176141.516608:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176141.516621:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176141.516627:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176141.516633:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.516640:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176141.516645:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.516649:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176141.516657:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176141.516660:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176141.516665:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176141.516671:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176141.516676:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176141.516681:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176141.516687:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.516691:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176141.516694:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.516697:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176141.516702:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176141.516706:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176141.516709:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176141.516713:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176141.516717:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176141.516721:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.516723:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176141.516727:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176141.516737:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176141.516742:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176141.516747:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176141.516749:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.516753:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176141.516756:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176141.516761:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176141.516770:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176141.516778:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176141.516789:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235455360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176141.516821:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176141.516827:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176141.516834:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235455360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176141.516863:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176141.516868:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176141.516872:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176141.516874:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176141.516877:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176141.516881:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176141.516884:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10815 00000400:00000200:3.0:1786176141.516890:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176141.516896:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176141.516900:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176141.516905:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.516909:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176141.516912:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.516915:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176141.516918:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176141.516920:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176141.516924:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176141.516930:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176141.516936:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176141.516942:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176141.516948:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.516954:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176141.516961:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.516970:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176141.516978:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176141.516988:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176141.517002:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176141.517010:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176141.517018:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176141.517021:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.517023:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176141.517028:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176141.517035:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.517040:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176141.517055:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176141.517064:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176141.517072:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176141.517078:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.517084:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176141.517088:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176141.517096:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176141.517102:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176141.517110:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176141.517121:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235455488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176141.517143:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176141.517149:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176141.517161:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235455488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176141.517178:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176141.517185:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176141.517189:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176141.517191:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176141.517194:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176141.517205:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176141.517209:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10815 00000400:00000200:3.0:1786176141.517215:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176141.517227:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176141.517233:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176141.517242:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.517249:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176141.517256:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.517265:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176141.517272:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176141.517277:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176141.517282:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176141.517290:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176141.517299:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176141.517307:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176141.517321:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.517326:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176141.517336:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.517340:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176141.517347:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176141.517351:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176141.517387:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176141.517391:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176141.517395:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176141.517399:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.517401:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176141.517404:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176141.517407:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.517411:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176141.517423:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176141.517432:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176141.517441:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176141.517448:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.517451:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176141.517457:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176141.517461:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176141.517470:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176141.517482:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176141.517502:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235455616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176141.517562:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176141.517586:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176141.517601:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235455616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176141.517642:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176141.517651:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176141.517658:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176141.517663:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176141.517668:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176141.517675:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176141.517687:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10815 00000800:00000200:2.1:1786176141.517691:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176141.517698:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000400:00000200:3.0:1786176141.517699:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176141.517711:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176141.517718:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176141.517730:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.517735:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176141.517741:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.517746:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176141.517754:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176141.517759:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176141.517764:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176141.517767:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176141.517775:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176141.517784:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176141.517791:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.517797:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176141.517804:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.517829:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176141.517838:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176141.517848:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176141.517854:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176141.517861:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176141.517866:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176141.517875:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.517884:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176141.517891:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176141.517903:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.517911:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176141.517919:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176141.517927:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176141.517936:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176141.517942:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.517946:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176141.517951:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176141.517954:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176141.517964:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176141.517973:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176141.517987:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235455744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176141.518010:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176141.518018:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176141.518029:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235455744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176141.518048:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176141.518059:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176141.518070:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176141.518073:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176141.518079:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176141.518086:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176141.518096:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10815 00000400:00000200:3.0:1786176141.518110:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176141.518119:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176141.518126:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176141.518152:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.518161:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176141.518167:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.518173:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176141.518180:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176141.518184:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176141.518189:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176141.518193:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176141.518215:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176141.518223:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176141.518233:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.518238:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176141.518246:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176141.518252:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176141.518259:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176141.518269:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176141.518280:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176141.518287:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176141.518295:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176141.518301:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.518306:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176141.518313:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176141.518323:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.518330:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176141.518342:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176141.518352:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176141.518361:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176141.518366:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176141.518372:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176141.518377:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176141.518384:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176141.518399:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176141.518405:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176141.518413:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235455872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176141.518512:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176141.518519:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176141.518557:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235455872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176141.518578:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176141.518590:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10826) 00000800:00000200:2.1:1786176141.518831:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176141.518835:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176141.519197:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176141.519201:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176141.519674:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176141.519677:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176141.520024:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176141.520027:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1786176144.460583:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176144.460593:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176144.460598:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176144.460623:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176144.460632:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176144.460651:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176144.460660:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176144.460666:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176144.460672:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176145.484532:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176145.484547:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176145.484552:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176145.484563:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176145.484581:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176145.484592:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176145.484600:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176145.484613:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176145.484636:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.508459:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176146.508470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176146.508473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176146.508480:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176146.508510:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176146.508515:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176146.508519:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176146.508522:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176146.508528:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176146.636511:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176146.636531:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176146.636536:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176146.636543:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176146.636551:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176146.636563:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10821 00000400:00000200:3.0:1786176146.636579:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176146.636590:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176146.636596:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176146.636602:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.636609:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176146.636615:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.636619:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176146.636625:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176146.636627:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176146.636631:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176146.636636:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176146.636639:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176146.636642:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176146.636647:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.636649:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176146.636652:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.636654:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176146.636658:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176146.636662:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176146.636665:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176146.636669:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176146.636673:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176146.636677:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.636679:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176146.636683:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176146.636693:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176146.636700:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176146.636705:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176146.636707:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.636712:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176146.636716:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176146.636721:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176146.636731:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176146.636739:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176146.636752:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235456000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176146.636767:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176146.636774:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176146.636781:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235456000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176146.636813:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176146.636817:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176146.636822:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176146.636824:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176146.636826:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176146.636830:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176146.636833:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10821 00000400:00000200:3.0:1786176146.636838:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176146.636842:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176146.636845:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176146.636849:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.636852:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176146.636856:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.636859:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176146.636863:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176146.636865:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176146.636868:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176146.636871:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176146.636874:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176146.636877:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176146.636881:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.636886:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176146.636891:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.636895:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176146.636899:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176146.636902:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176146.636905:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176146.636908:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176146.636915:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176146.636918:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.636921:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176146.636927:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176146.636936:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.636941:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176146.636950:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176146.636957:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176146.636960:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176146.636963:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.636970:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176146.636974:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176146.636980:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176146.636996:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176146.637010:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176146.637030:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235456128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176146.637083:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176146.637089:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176146.637105:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235456128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176146.637159:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176146.637168:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176146.637177:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176146.637182:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176146.637188:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176146.637200:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176146.637207:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10821 00000400:00000200:3.0:1786176146.637227:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176146.637239:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176146.637246:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176146.637269:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.637284:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176146.637294:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.637309:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176146.637317:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176146.637322:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176146.637330:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176146.637337:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176146.637340:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176146.637344:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176146.637396:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.637400:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176146.637402:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.637405:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176146.637407:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176146.637411:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176146.637414:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176146.637418:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176146.637424:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176146.637427:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.637429:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176146.637433:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176146.637437:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.637439:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176146.637444:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176146.637449:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176146.637453:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176146.637455:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.637465:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176146.637468:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176146.637472:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176146.637477:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176146.637516:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176146.637531:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235456256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176146.637566:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176146.637574:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176146.637592:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235456256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176146.637627:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176146.637637:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176146.637650:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176146.637653:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176146.637656:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176146.637660:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176146.637664:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10821 00000400:00000200:3.0:1786176146.637679:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176146.637694:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176146.637705:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176146.637725:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.637731:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176146.637739:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.637744:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176146.637748:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176146.637756:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176146.637766:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176146.637774:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176146.637794:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176146.637806:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000200:2.1:1786176146.637808:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786176146.637820:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:2.1:1786176146.637823:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1786176146.637832:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176146.637843:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.637852:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176146.637860:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176146.637877:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176146.637882:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176146.637884:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176146.637889:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176146.637904:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.637909:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176146.637919:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176146.637926:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.637935:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176146.637944:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786176146.637950:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176146.637965:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176146.637969:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.637982:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176146.637986:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176146.637994:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176146.638007:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176146.638016:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176146.638027:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235456384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176146.638104:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176146.638120:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176146.638127:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235456384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176146.638193:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176146.638208:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176146.638217:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176146.638222:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176146.638226:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176146.638232:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176146.638247:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10821 00000400:00000200:3.0:1786176146.638268:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176146.638281:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176146.638287:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176146.638305:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.638310:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176146.638315:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.638324:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176146.638327:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176146.638330:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176146.638332:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176146.638336:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176146.638340:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176146.638351:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176146.638398:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.638403:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176146.638411:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176146.638415:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176146.638419:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176146.638424:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176146.638428:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176146.638430:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176146.638435:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176146.638441:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.638446:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176146.638450:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176146.638458:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.638467:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176146.638476:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786176146.638506:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176146.638517:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176146.638526:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176146.638543:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176146.638551:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176146.638560:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176146.638570:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176146.638583:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176146.638596:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235456512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176146.638625:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176146.638636:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176146.638662:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235456512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176146.638730:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176146.638740:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10831) 00000800:00000200:2.1:1786176146.639099:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176146.639102:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176146.639549:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176146.639553:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176146.639923:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176146.639927:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176146.640258:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176146.640261:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1786176147.532457:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176147.532466:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176147.532468:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176147.532486:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176147.532493:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176147.532508:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176147.532527:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176147.532531:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176147.532535:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176148.172557:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176148.172572:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176148.172576:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176148.172584:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176148.172590:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176148.172595:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176148.172601:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176148.172605:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176148.172611:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176148.556510:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176148.556521:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176148.556542:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176148.556550:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176148.556561:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176148.556566:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176148.556572:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176148.556578:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176148.556592:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176149.580450:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176149.580479:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176149.580482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176149.580500:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176149.580505:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176149.580511:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176149.580515:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176149.580519:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176149.580524:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176150.604471:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176150.604509:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176150.604511:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176150.604517:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176150.604521:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176150.604527:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176150.604532:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176150.604536:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176150.604541:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176151.628553:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176151.628564:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176151.628568:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176151.628575:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176151.628583:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176151.628589:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176151.628597:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176151.628602:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176151.628605:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176151.757714:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176151.757726:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176151.758719:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176151.758721:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176151.759063:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176151.759066:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176151.759513:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176151.759517:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176151.759834:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176151.759839:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1786176152.652517:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176152.652528:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176152.652534:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176152.652544:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176152.652551:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176152.652556:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176152.652559:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176152.652562:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176152.652567:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176153.676497:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176153.676507:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176153.676513:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176153.676518:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176153.676527:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176153.676532:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176153.676536:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176153.676539:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176153.676542:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176154.700520:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176154.700529:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176154.700531:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176154.700539:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176154.700545:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176154.700551:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176154.700555:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176154.700561:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176154.700566:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176155.724541:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176155.724574:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176155.724579:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176155.724596:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176155.724616:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176155.724633:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176155.724642:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176155.724648:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176155.724660:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176156.365354:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176156.365470:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176156.365476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176156.365489:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176156.365497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176156.365506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176156.365511:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176156.365517:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176156.365524:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176156.748502:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176156.748519:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176156.748524:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176156.748537:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176156.748547:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176156.748556:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176156.748560:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176156.748564:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176156.748569:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176156.877674:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176156.877686:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176156.878921:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176156.878924:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176156.879292:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176156.879296:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176156.879766:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176156.879769:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176156.880113:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176156.880116:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1786176157.388700:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176157.388716:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176157.388721:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176157.388732:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176157.388740:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176157.388748:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176157.388755:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176157.388763:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176157.388772:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176157.772479:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176157.772491:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176157.772494:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176157.772502:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176157.772513:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176157.772522:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176157.772533:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176157.772550:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176157.772556:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176158.796503:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176158.796513:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176158.796518:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176158.796528:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176158.796536:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176158.796542:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176158.796553:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176158.796560:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176158.796567:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176159.820477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176159.820484:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176159.820488:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176159.820495:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176159.820501:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176159.820506:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176159.820509:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176159.820512:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176159.820514:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176160.844433:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176160.844439:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176160.844441:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176160.844445:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176160.844449:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176160.844453:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176160.844458:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176160.844463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176160.844467:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1786176161.420445:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:3.0:1786176161.420453:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:3.0:1786176161.420458:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:3.0:1786176161.868426:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176161.868433:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176161.868435:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176161.868441:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176161.868446:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176161.868451:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176161.868454:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176161.868458:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176161.868462:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176161.997672:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176161.997682:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176161.998579:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176161.998582:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176161.999134:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176161.999137:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176161.999598:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176161.999601:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176161.999950:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176161.999955:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1786176162.892468:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176162.892476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176162.892478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176162.892483:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176162.892488:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176162.892491:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176162.892497:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176162.892500:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176162.892503:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176163.916455:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176163.916466:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176163.916470:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176163.916489:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176163.916495:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176163.916501:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176163.916505:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176163.916513:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176163.916521:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176164.556451:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176164.556458:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176164.556460:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176164.556464:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176164.556468:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176164.556470:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176164.556473:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176164.556475:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176164.556478:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176164.940471:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176164.940480:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176164.940483:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176164.940487:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176164.940494:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176164.940499:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176164.940503:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176164.940506:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176164.940508:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176165.580450:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176165.580457:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176165.580461:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176165.580466:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176165.580469:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176165.580473:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176165.580476:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176165.580479:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176165.580486:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176165.964451:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176165.964459:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176165.964461:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176165.964466:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176165.964470:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176165.964473:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176165.964476:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176165.964479:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176165.964483:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176166.988425:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176166.988432:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176166.988435:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176166.988439:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176166.988444:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176166.988448:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176166.988453:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176166.988464:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176166.988467:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176167.116990:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176167.116994:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176167.117627:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176167.117629:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176167.117866:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176167.117868:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176167.118051:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176167.118053:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176167.118225:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176167.118227:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1786176168.012440:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176168.012449:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176168.012451:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176168.012457:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176168.012461:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176168.012467:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176168.012472:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176168.012476:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176168.012479:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176169.036458:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176169.036468:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176169.036471:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176169.036477:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176169.036483:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176169.036489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176169.036496:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176169.036503:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176169.036515:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176170.060446:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176170.060455:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176170.060459:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176170.060463:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176170.060469:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176170.060473:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176170.060479:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176170.060482:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176170.060484:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176171.084447:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176171.084455:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176171.084458:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176171.084463:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176171.084467:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176171.084471:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176171.084474:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176171.084478:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176171.084480:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176172.108435:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176172.108443:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176172.108445:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176172.108452:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176172.108458:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176172.108465:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176172.108473:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176172.108480:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176172.108487:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176172.237134:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176172.237536:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176172.237866:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176172.238064:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176172.238254:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176173.132440:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176173.132448:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176173.132452:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176173.132456:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176173.132463:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176173.132467:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176173.132470:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176173.132472:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176173.132475:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176173.772486:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176173.772497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176173.772500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176173.772505:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176173.772511:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176173.772516:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176173.772519:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176173.772522:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176173.772524:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176174.156445:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176174.156472:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176174.156475:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176174.156483:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176174.156489:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176174.156495:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176174.156500:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176174.156504:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176174.156506:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176174.796504:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176174.796519:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176174.796523:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176174.796532:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176174.796539:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176174.796546:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176174.796552:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176174.796556:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176174.796574:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176175.180473:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176175.180482:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176175.180486:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176175.180491:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176175.180498:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176175.180503:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176175.180507:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176175.180509:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176175.180512:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1786176175.308617:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:3.0:1786176175.308652:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1786176175.308660:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176175.308663:0:2381:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:3.0:1786176175.308668:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176175.308671:0:2381:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1786176175.308675:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176175.308678:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:3.0:1786176175.308683:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176175.308688:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176175.308694:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e2a8 00000400:00000010:3.0:1786176175.308697:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e2a8. 00000100:00000001:3.0:1786176175.308703:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176175.308731:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09317800 x1872930235459200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176175.308764:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176175.308771:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317800 x1872930235459200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176175.308784:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176175.308787:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176175.308800:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176175.308805:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:3.0:1786176175.308811:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176175.308816:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176175.308822:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:3.0:1786176175.308825:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:3.0:1786176175.308830:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176175.308837:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09314380 x1872930235459328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176175.308867:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176175.308874:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314380 x1872930235459328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176175.308888:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176175.308893:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176175.308903:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176175.308911:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:3.0:1786176175.308917:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176175.308924:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176175.308946:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:3.0:1786176175.308952:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:3.0:1786176175.308958:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176175.308968:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09316300 x1872930235459456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176175.308996:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176175.309003:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316300 x1872930235459456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176175.309019:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176175.309024:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176175.309035:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176175.309039:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:3.0:1786176175.309045:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176175.309056:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176175.309064:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:3.0:1786176175.309069:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:3.0:1786176175.309074:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176175.309083:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09316a00 x1872930235459584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176175.309101:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176175.309110:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316a00 x1872930235459584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176175.309125:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176175.309131:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176175.309143:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176175.309149:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:3.0:1786176175.309155:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176175.309162:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176175.309169:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e3b8 00000400:00000010:3.0:1786176175.309173:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e3b8. 00000100:00000001:3.0:1786176175.309177:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176175.309184:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09315c00 x1872930235459712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176175.309200:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176175.309221:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315c00 x1872930235459712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176175.309239:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176175.309246:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1786176175.820462:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176175.820472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176175.820474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176175.820482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176175.820487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176175.820492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176175.820495:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176175.820499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176175.820503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176176.204429:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176176.204438:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176176.204441:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176176.204448:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176176.204453:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176176.204457:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176176.204460:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176176.204463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176176.204470:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176176.844453:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176176.844461:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176176.844463:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176176.844468:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176176.844472:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176176.844476:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176176.844478:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176176.844481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176176.844484:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176177.228509:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176177.228520:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176177.228528:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176177.228542:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176177.228549:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176177.228554:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176177.228558:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176177.228561:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176177.228565:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176177.357330:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176177.357336:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176177.358159:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176177.358164:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176177.358715:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176177.358719:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176177.359066:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176177.359070:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176177.359349:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176177.359353:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786176177.868440:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176177.868451:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176177.868454:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176177.868460:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176177.868465:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176177.868469:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176177.868471:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176177.868474:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176177.868477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176178.252484:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176178.252496:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176178.252499:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176178.252509:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176178.252517:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176178.252526:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176178.252534:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176178.252554:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176178.252562:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176178.892482:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176178.892494:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176178.892498:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176178.892517:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176178.892522:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176178.892529:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176178.892533:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176178.892536:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176178.892540:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176179.276468:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176179.276476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176179.276478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176179.276484:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176179.276487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176179.276491:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176179.276493:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176179.276496:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176179.276499:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176179.916449:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176179.916461:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176179.916464:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176179.916471:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176179.916477:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176179.916481:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176179.916484:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176179.916488:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176179.916492:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176180.300452:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176180.300462:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176180.300464:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176180.300472:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176180.300478:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176180.300483:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176180.300489:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176180.300492:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176180.300495:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176180.940453:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176180.940462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176180.940463:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176180.940467:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176180.940472:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176180.940475:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176180.940477:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176180.940479:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176180.940482:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176181.324420:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176181.324428:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176181.324429:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176181.324434:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176181.324437:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176181.324440:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176181.324442:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176181.324445:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176181.324447:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176181.964476:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176181.964489:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176181.964493:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176181.964501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176181.964509:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176181.964516:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176181.964521:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176181.964526:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176181.964531:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176182.348451:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176182.348458:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176182.348462:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176182.348466:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176182.348472:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176182.348476:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176182.348480:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176182.348483:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176182.348485:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176182.477231:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176182.477238:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786176182.478113:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176182.478115:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786176182.478335:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176182.478337:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786176182.478551:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176182.478554:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786176182.478802:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176182.478804:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000100:00000001:3.0:1786176182.988485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176182.988495:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176182.988498:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176182.988507:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176182.988514:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176182.988520:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176182.988523:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176182.988539:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176182.988545:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176183.372459:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176183.372480:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176183.372482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176183.372487:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176183.372491:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176183.372495:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176183.372498:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176183.372501:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176183.372504:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176184.012502:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176184.012514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176184.012517:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176184.012527:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176184.012537:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176184.012543:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176184.012547:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176184.012551:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176184.012556:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176184.396519:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176184.396531:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176184.396535:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176184.396543:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176184.396551:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176184.396558:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176184.396563:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176184.396568:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176184.396571:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176185.036470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176185.036483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176185.036486:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176185.036492:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176185.036498:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176185.036503:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176185.036506:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176185.036508:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176185.036512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176185.420434:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176185.420441:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176185.420443:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176185.420448:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176185.420452:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176185.420456:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176185.420458:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176185.420461:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176185.420464:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176186.060457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176186.060464:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176186.060466:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176186.060472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176186.060476:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176186.060479:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176186.060481:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176186.060483:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176186.060486:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176186.444414:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176186.444419:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176186.444421:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176186.444424:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176186.444427:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176186.444430:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176186.444434:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176186.444438:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176186.444441:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176187.084516:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176187.084542:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176187.084545:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176187.084551:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176187.084558:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176187.084561:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176187.084564:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176187.084566:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176187.084568:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176187.468429:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176187.468437:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176187.468440:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176187.468445:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176187.468450:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176187.468454:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176187.468456:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176187.468458:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176187.468462:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176187.597569:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176187.597578:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176187.598496:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176187.598499:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176187.598857:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176187.598861:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176187.599242:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176187.599245:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176187.599624:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176187.599627:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1786176188.108476:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176188.108485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176188.108487:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176188.108493:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176188.108497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176188.108501:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176188.108504:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176188.108506:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176188.108510:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176189.132445:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176189.132454:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176189.132457:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176189.132464:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176189.132468:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176189.132472:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176189.132475:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176189.132478:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176189.132482:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176190.156494:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176190.156509:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176190.156514:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176190.156525:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176190.156535:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176190.156544:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176190.156550:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176190.156557:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176190.156563:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176190.540685:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176190.540694:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176190.540702:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176191.180453:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176191.180462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176191.180465:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176191.180470:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176191.180485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176191.180489:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176191.180492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176191.180495:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176191.180499:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176192.204557:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176192.204582:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176192.204588:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176192.204599:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176192.204606:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176192.204612:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176192.204617:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176192.204621:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176192.204626:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176192.588626:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176192.588632:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176192.588637:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176192.717228:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176192.717235:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176192.718010:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176192.718015:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176192.718400:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176192.718403:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176192.718708:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176192.718711:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176192.719267:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176192.719270:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1786176193.228521:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176193.228532:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176193.228535:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176193.228542:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176193.228549:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176193.228553:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176193.228558:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176193.228561:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176193.228566:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176194.252465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176194.252476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176194.252480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176194.252487:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176194.252496:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176194.252501:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176194.252505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176194.252509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176194.252513:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176194.636649:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176194.636657:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176194.636663:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176195.276524:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176195.276538:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176195.276543:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176195.276553:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176195.276563:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176195.276571:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176195.276577:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176195.276583:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176195.276590:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176196.300534:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176196.300552:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176196.300557:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176196.300569:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176196.300577:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176196.300584:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176196.300589:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176196.300595:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176196.300602:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176196.684705:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176196.684711:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176196.684715:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176197.708793:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176197.708800:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176197.708805:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176197.837673:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176197.837681:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176197.838790:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176197.838793:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176197.839286:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176197.839289:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176197.839674:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176197.839677:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176197.840123:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176197.840129:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000020:00000001:3.0:1786176198.732614:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176198.732620:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176198.732625:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176199.756527:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176199.756531:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176199.756534:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176200.780620:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176200.780625:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176200.780629:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176201.804532:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176201.804536:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176201.804539:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176202.828526:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176202.828529:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176202.828532:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176202.957217:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176202.957221:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176202.957903:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176202.957905:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176202.958232:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176202.958235:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176202.958625:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176202.958629:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176202.959012:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176202.959016:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000020:00000001:3.0:1786176203.852552:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176203.852556:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176203.852559:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176204.876685:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176204.876701:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176204.876707:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176205.900535:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176205.900539:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176205.900542:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176206.924694:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176206.924701:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176206.924706:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176207.948692:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176207.948700:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176207.948706:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176208.077004:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176208.077011:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176208.077628:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176208.077630:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176208.077847:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176208.077849:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176208.078086:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176208.078088:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176208.078404:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176208.078406:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000020:00000001:3.0:1786176208.972530:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176208.972534:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176208.972536:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176209.996630:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176209.996634:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176209.996638:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176211.020624:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176211.020633:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176211.020639:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176212.044563:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176212.044570:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176212.044574:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176213.068638:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176213.068643:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176213.068647:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176213.197528:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176213.197535:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176213.198853:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176213.198877:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176213.199527:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176213.199530:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176213.200065:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176213.200068:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176213.200481:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176213.200484:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000020:00000001:3.0:1786176214.092505:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176214.092509:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176214.092511:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176215.116524:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176215.116527:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176215.116530:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176215.756439:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176215.756447:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176215.756449:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176215.756452:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176215.756456:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176215.756459:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176215.756460:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176215.756462:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176215.756465:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176216.140501:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176216.140505:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176216.140508:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176216.780476:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176216.780486:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176216.780489:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176216.780497:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176216.780504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176216.780511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176216.780515:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176216.780519:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176216.780535:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176217.164628:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176217.164635:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176217.164641:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176217.804534:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176217.804545:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176217.804549:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176217.804557:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176217.804564:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176217.804569:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176217.804572:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176217.804575:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176217.804579:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176218.188680:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176218.188689:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176218.188696:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176218.317389:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176218.317396:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176218.318753:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176218.318760:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176218.319210:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176218.319213:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176218.319650:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176218.319653:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176218.320030:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176218.320033:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1786176218.828462:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176218.828472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176218.828475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176218.828483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176218.828488:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176218.828492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176218.828496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176218.828499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176218.828503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176219.212748:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176219.212753:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176219.212757:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176219.852497:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176219.852506:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176219.852509:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176219.852515:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176219.852519:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176219.852523:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176219.852526:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176219.852529:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176219.852532:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176220.236622:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176220.236626:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176220.236630:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176220.876561:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176220.876576:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176220.876580:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176220.876591:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176220.876599:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176220.876606:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176220.876639:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176220.876646:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176220.876653:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176221.260703:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176221.260709:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176221.260714:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176221.900478:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176221.900488:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176221.900491:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176221.900498:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176221.900502:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176221.900507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176221.900511:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176221.900515:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176221.900519:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176222.284508:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176222.284512:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176222.284515:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176222.924657:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176222.924671:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176222.924674:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176222.924679:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176222.924683:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176222.924687:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176222.924690:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176222.924693:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176222.924697:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176223.308650:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176223.308657:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176223.308663:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176223.437453:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176223.437460:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786176223.438734:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176223.438737:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786176223.439191:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176223.439194:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786176223.439576:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176223.439580:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786176223.440119:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176223.440122:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000020:00000001:3.0:1786176224.332587:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176224.332593:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176224.332598:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176225.356705:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176225.356712:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176225.356717:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176226.380530:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176226.380534:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176226.380538:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176227.404782:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176227.404788:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176227.404792:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176228.428528:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176228.428532:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176228.428535:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176228.557453:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176228.557480:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176228.558405:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176228.558406:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176228.558769:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176228.558771:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176228.559048:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176228.559050:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176228.559355:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176228.559357:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000020:00000001:3.0:1786176229.452512:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176229.452516:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176229.452519:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176230.476495:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176230.476499:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176230.476502:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176231.500587:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176231.500593:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176231.500598:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176232.524676:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176232.524685:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176232.524692:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176233.548627:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176233.548634:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176233.548640:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176233.677708:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176233.677724:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176233.679154:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176233.679158:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176233.679625:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176233.679628:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176233.680145:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176233.680149:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176233.680608:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176233.680611:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000020:00000001:3.0:1786176234.572663:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176234.572669:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176234.572674:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176235.596550:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176235.596555:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176235.596558:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176236.620583:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176236.620588:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176236.620591:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176237.644663:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176237.644666:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176237.644670:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176238.668680:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176238.668684:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176238.668687:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176238.797330:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176238.797991:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176238.798508:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176238.798763:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176238.799001:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:3.0:1786176239.692558:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176239.692561:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176239.692564:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176240.716642:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176240.716647:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176240.716651:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176241.740551:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176241.740557:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176241.740607:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1786176241.868732:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:3.0:1786176241.868780:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1786176241.868790:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176241.868795:0:2380:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:3.0:1786176241.868800:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176241.868805:0:2380:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1786176241.868812:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176241.868830:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:3.0:1786176241.868839:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176241.868846:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176241.868855:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e440 00000400:00000010:3.0:1786176241.868859:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e440. 00000100:00000001:3.0:1786176241.868867:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176241.868906:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31736680 x1872930235467520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176241.868941:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176241.868948:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235467520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176241.868960:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176241.868963:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176241.868994:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176241.868999:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:3.0:1786176241.869006:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176241.869012:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176241.869018:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:3.0:1786176241.869021:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:3.0:1786176241.869027:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176241.869034:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31734e00 x1872930235467648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176241.869049:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176241.869062:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734e00 x1872930235467648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176241.869080:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176241.869086:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176241.869105:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176241.869111:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:3.0:1786176241.869121:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176241.869147:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176241.869186:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eb28 00000400:00000010:3.0:1786176241.869197:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eb28. 00000100:00000001:3.0:1786176241.869211:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176241.869229:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735180 x1872930235467776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176241.869284:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176241.869302:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735180 x1872930235467776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176241.869317:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176241.869320:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176241.869347:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176241.869457:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:3.0:1786176241.869471:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176241.869543:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176241.869557:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:3.0:1786176241.869588:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:3.0:1786176241.869598:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176241.869630:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31736a00 x1872930235467904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176241.869667:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176241.870098:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235467904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176241.870130:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176241.870137:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176241.870162:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176241.870174:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:3.0:1786176241.870185:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176241.870198:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176241.870208:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e990 00000400:00000010:3.0:1786176241.870217:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e990. 00000100:00000001:3.0:1786176241.870231:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176241.870247:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31737100 x1872930235468032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176241.870284:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176241.870297:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235468032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176241.870326:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176241.870334:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1786176242.764548:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176242.764551:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176242.764568:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176243.788668:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176243.788680:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176243.788685:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176243.917681:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176243.917694:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176243.918792:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176243.918797:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176243.919287:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176243.919291:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176243.919794:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176243.919797:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176243.920257:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176243.920259:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000020:00000001:3.0:1786176244.812678:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176244.812689:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176244.812701:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176245.836695:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176245.836701:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176245.836706:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176246.860697:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176246.862655:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176246.862662:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176247.884583:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176247.884605:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176247.884609:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176248.915669:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176248.915673:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176248.915675:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176249.038156:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176249.038173:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176249.038846:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176249.038849:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176249.040103:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176249.040105:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176249.040282:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176249.040284:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176249.040554:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176249.040556:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000020:00000001:3.0:1786176249.932638:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176249.932642:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176249.932645:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176250.956698:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176250.956746:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176250.956751:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176251.980610:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176251.980615:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176251.980619:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176253.004654:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176253.004659:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176253.004663:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176254.028644:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176254.028652:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176254.028658:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176254.157598:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176254.157605:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176254.159144:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176254.159158:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176254.159681:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176254.159682:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176254.159917:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176254.159919:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176254.160093:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176254.160095:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000020:00000001:3.0:1786176255.052618:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176255.052622:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176255.052626:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176256.076511:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176256.076515:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176256.076518:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176257.100813:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176257.100821:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176257.100826:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176258.124538:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176258.124543:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176258.124546:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176259.148554:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176259.148558:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176259.148560:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176259.277461:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176259.277472:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176259.278234:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176259.278260:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176259.278667:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176259.278670:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176259.278905:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176259.278907:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176259.279265:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176259.279269:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000020:00000001:3.0:1786176260.172717:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176260.172724:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176260.172729:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176261.196518:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176261.196522:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176261.196525:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176261.836481:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176261.836488:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176261.836489:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176261.836494:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176261.836497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176261.836500:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176261.836502:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176261.836504:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176261.836506:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176262.220580:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176262.220583:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176262.220586:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176262.860557:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176262.860567:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176262.860569:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176262.860575:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176262.860581:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176262.860586:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176262.860591:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176262.860594:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176262.860599:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176263.244660:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176263.244666:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176263.244671:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176263.884484:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176263.884497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176263.884502:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176263.884513:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176263.884523:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176263.884530:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176263.884537:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176263.884542:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176263.884549:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176264.268646:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176264.268650:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176264.268654:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176264.397509:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176264.397518:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786176264.398501:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176264.398503:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786176264.398785:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176264.398789:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786176264.399101:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176264.399104:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786176264.399610:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176264.399613:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00000001:3.0:1786176264.908495:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176264.908506:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176264.908508:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176264.908515:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176264.908520:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176264.908525:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176264.908528:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176264.908531:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176264.908534:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176265.292573:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176265.292579:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176265.292584:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176265.932423:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176265.932433:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176265.932436:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176265.932442:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176265.932446:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176265.932450:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176265.932453:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176265.932456:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176265.932459:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176266.316505:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176266.316510:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176266.316514:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176267.340766:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176267.340775:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176267.340784:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176268.364663:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176268.364672:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176268.364678:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176269.388679:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176269.388686:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176269.388691:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176269.517035:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176269.517042:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176269.517849:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176269.517851:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176269.518091:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176269.518094:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176269.518435:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176269.518436:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176269.518741:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176269.518743:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000020:00000001:3.0:1786176270.412554:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176270.412560:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176270.412564:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176271.436535:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176271.436542:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176271.436545:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176272.460722:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176272.460729:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176272.460734:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176273.484526:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176273.484531:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176273.484537:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176274.508632:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176274.508639:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176274.508645:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176274.636510:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176274.636523:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176274.636527:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176274.636532:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176274.636539:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176274.636549:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10949 00000400:00000200:3.0:1786176274.636562:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176274.636570:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176274.636575:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176274.636580:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.636584:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176274.636588:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.636593:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176274.636598:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176274.636601:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176274.636605:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176274.636610:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176274.636613:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176274.636617:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176274.636622:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.636625:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176274.636628:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.636631:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176274.636636:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176274.636641:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176274.636645:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176274.636649:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176274.636653:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176274.636657:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.636661:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176274.636665:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176274.636676:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786176274.636683:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176274.636688:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176274.636690:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.636694:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176274.636698:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176274.636704:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176274.636716:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176274.636727:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176274.636743:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235472000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176274.636758:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176274.636765:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176274.636773:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235472000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176274.636805:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176274.636810:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176274.636816:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176274.636818:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176274.636821:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176274.636825:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176274.636830:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10949 00000400:00000200:3.0:1786176274.636838:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176274.636845:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176274.636849:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176274.636859:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.636868:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176274.636880:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.636889:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176274.636898:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176274.636903:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176274.636910:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176274.636916:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176274.636925:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176274.636933:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176274.636947:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.636957:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176274.636964:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.636969:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176274.636981:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176274.636994:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176274.637003:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176274.637010:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176274.637018:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176274.637030:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.637038:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176274.637046:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176274.637055:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.637064:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176274.637078:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176274.637091:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176274.637108:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176274.637118:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.637128:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176274.637135:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176274.637143:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176274.637155:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176274.637167:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176274.637183:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235472128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176274.637210:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176274.637217:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176274.637229:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235472128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1786176274.637275:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176274.637280:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1786176274.637283:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176274.637292:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176274.637304:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176274.637310:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176274.637316:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176274.637324:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176274.637335:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10949 00000400:00000200:3.0:1786176274.637351:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176274.637415:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176274.637425:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176274.637435:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.637446:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176274.637456:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.637468:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176274.637477:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176274.637483:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176274.637487:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176274.637490:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176274.637494:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176274.637503:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176274.637509:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.637517:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176274.637529:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.637539:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176274.637548:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176274.637562:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176274.637569:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176274.637578:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176274.637586:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176274.637596:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.637605:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176274.637611:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176274.637620:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.637627:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176274.637639:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176274.637645:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176274.637656:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176274.637661:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.637669:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176274.637674:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176274.637679:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176274.637689:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176274.637700:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176274.637714:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235472256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176274.637748:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176274.637755:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176274.637771:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235472256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176274.637807:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176274.637817:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176274.637832:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176274.637837:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176274.637843:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176274.637856:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176274.637866:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10949 00000400:00000200:3.0:1786176274.637879:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176274.637895:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176274.637905:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176274.637918:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.637928:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176274.637936:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.637947:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176274.637957:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176274.637961:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176274.637964:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176274.637967:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176274.637970:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176274.637974:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176274.637980:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.637983:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176274.637988:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.637991:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176274.637994:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176274.637998:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176274.638002:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176274.638005:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176274.638010:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176274.638013:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.638016:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176274.638019:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176274.638022:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.638025:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176274.638031:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786176274.638035:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176274.638040:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176274.638042:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.638045:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176274.638047:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176274.638049:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176274.638055:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176274.638060:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176274.638068:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235472384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176274.638079:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176274.638082:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176274.638087:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235472384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176274.638100:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176274.638104:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176274.638109:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176274.638111:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176274.638114:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176274.638118:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176274.638121:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10949 00000400:00000200:3.0:1786176274.638126:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176274.638131:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176274.638134:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176274.638138:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.638141:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176274.638145:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.638148:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176274.638151:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176274.638153:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176274.638156:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176274.638158:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176274.638161:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176274.638165:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176274.638170:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.638173:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176274.638176:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176274.638180:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176274.638183:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176274.638187:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176274.638191:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176274.638194:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176274.638198:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176274.638201:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.638203:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176274.638206:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176274.638209:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.638212:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176274.638217:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786176274.638221:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176274.638224:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176274.638226:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176274.638230:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176274.638232:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176274.638234:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176274.638240:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176274.638244:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176274.638279:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235472512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176274.638289:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176274.638309:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176274.638314:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235472512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176274.638324:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176274.638330:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10959) 00000800:00000200:2.1:1786176274.639545:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176274.639547:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176274.639849:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176274.639852:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176274.641543:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176274.641546:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176274.641818:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176274.641820:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000020:00000001:3.0:1786176275.532476:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176275.532480:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176275.532484:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176276.556651:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176276.556656:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176276.556660:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176277.580608:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176277.580613:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176277.580616:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176278.604661:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176278.604666:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176278.604672:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176279.628674:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176279.628684:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176279.628692:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176279.756486:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176279.756506:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176279.756512:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176279.756519:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176279.756527:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176279.756539:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10954 00000400:00000200:3.0:1786176279.756555:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176279.756565:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176279.756571:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176279.756577:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.756583:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176279.756588:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.756592:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176279.756599:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176279.756603:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176279.756608:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176279.756614:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176279.756618:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176279.756623:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176279.756628:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.756631:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176279.756634:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.756637:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176279.756643:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176279.756648:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176279.756653:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176279.756674:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176279.756680:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176279.756686:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.756689:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176279.756694:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176279.756710:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786176279.756716:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176279.756724:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176279.756727:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.756733:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176279.756737:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176279.756745:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176279.756758:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176279.756769:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176279.756786:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235472640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176279.756803:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176279.756823:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176279.756832:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235472640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176279.756858:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176279.756862:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176279.756866:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176279.756868:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176279.756870:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176279.756890:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176279.756895:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10954 00000400:00000200:3.0:1786176279.756904:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176279.756913:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176279.756920:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176279.756928:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.756934:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176279.756939:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.756945:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176279.756952:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176279.756957:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176279.756966:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176279.756997:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176279.757006:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176279.757024:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176279.757048:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.757061:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176279.757075:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.757088:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176279.757112:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176279.757131:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176279.757144:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176279.757151:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176279.757170:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176279.757182:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.757193:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176279.757220:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176279.757251:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.757255:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176279.757270:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176279.757277:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176279.757285:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176279.757287:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.757294:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176279.757297:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176279.757301:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176279.757323:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176279.757341:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176279.757426:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235472768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176279.757447:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176279.757456:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176279.757480:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235472768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176279.757534:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176279.757542:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176279.757554:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176279.757556:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176279.757558:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176279.757567:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176279.757580:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10954 00000400:00000200:3.0:1786176279.757634:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176279.757649:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176279.757662:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176279.757679:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.757693:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176279.757706:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.757720:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176279.757731:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176279.757744:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176279.757755:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:2.1:1786176279.757755:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786176279.757769:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000800:00000200:2.1:1786176279.757770:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000020:00000001:3.0:1786176279.757787:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176279.757798:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176279.757809:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.757823:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176279.757839:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.757857:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176279.757864:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176279.757898:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176279.757913:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176279.757929:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176279.757941:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176279.757955:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.757969:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176279.757996:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176279.758008:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.758013:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176279.758027:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176279.758039:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176279.758047:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176279.758051:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.758056:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176279.758058:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176279.758062:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176279.758069:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176279.758081:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176279.758098:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235472896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176279.758120:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176279.758124:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176279.758142:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235472896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176279.758178:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176279.758188:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176279.758221:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176279.758223:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176279.758227:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176279.758234:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176279.758239:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10954 00000400:00000200:3.0:1786176279.758248:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176279.758257:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176279.758264:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176279.758272:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.758282:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176279.758288:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.758299:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176279.758308:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176279.758313:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176279.758321:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176279.758327:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176279.758333:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176279.758341:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176279.758351:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.758395:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176279.758405:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.758422:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176279.758429:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176279.758438:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176279.758466:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176279.758476:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176279.758488:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176279.758498:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.758502:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176279.758512:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176279.758521:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.758547:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176279.758555:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176279.758560:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176279.758568:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176279.758569:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.758574:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176279.758577:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176279.758581:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176279.758592:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176279.758602:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176279.758616:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235473024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176279.758639:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176279.758646:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176279.758666:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235473024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176279.758728:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176279.758745:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176279.758762:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176279.758772:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176279.758780:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176279.758809:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176279.758825:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10954 00000400:00000200:3.0:1786176279.758838:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176279.758864:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176279.758869:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176279.758884:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.758898:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176279.758909:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.758920:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176279.758931:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176279.758945:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176279.758953:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176279.758959:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176279.758968:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176279.758979:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176279.758996:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.759002:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176279.759009:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176279.759021:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176279.759036:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176279.759046:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176279.759049:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176279.759051:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176279.759057:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176279.759059:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.759061:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176279.759064:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176279.759067:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.759068:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176279.759073:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176279.759075:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176279.759078:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176279.759081:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176279.759083:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176279.759088:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176279.759101:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176279.759130:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176279.759141:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176279.759173:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235473152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176279.759195:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176279.759218:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176279.759230:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235473152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176279.759281:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176279.759293:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10964) 00000800:00000200:2.1:1786176279.759563:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176279.759569:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176279.759972:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176279.759976:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176279.760572:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176279.760578:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176279.761084:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176279.761089:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000020:00000001:3.0:1786176280.652611:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176280.652621:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176280.652628:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176281.676497:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176281.676502:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176281.676506:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176282.700547:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176282.700550:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176282.700553:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176283.724573:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176283.724577:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176283.724587:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176284.748549:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176284.748552:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176284.748555:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176284.876420:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176284.876429:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176284.876431:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176284.876434:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176284.876438:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176284.876443:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10959 00000400:00000200:3.0:1786176284.876450:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176284.876455:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176284.876457:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176284.876460:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876463:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176284.876465:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876466:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176284.876469:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176284.876471:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176284.876473:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176284.876476:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176284.876477:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176284.876479:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176284.876481:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876483:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176284.876484:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876486:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176284.876488:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176284.876490:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876492:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176284.876494:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176284.876496:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176284.876499:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.876500:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176284.876502:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176284.876509:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176284.876512:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176284.876515:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176284.876516:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.876518:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176284.876520:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176284.876524:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176284.876530:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176284.876535:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176284.876543:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235473280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176284.876551:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176284.876554:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176284.876558:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235473280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176284.876574:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176284.876576:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176284.876579:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176284.876580:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176284.876581:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176284.876583:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176284.876585:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10959 00000400:00000200:3.0:1786176284.876587:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176284.876589:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176284.876591:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176284.876594:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876596:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176284.876598:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876600:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176284.876601:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176284.876602:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176284.876603:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176284.876605:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176284.876607:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176284.876611:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176284.876616:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876618:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176284.876622:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876624:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176284.876629:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176284.876635:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876642:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176284.876645:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176284.876651:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176284.876656:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.876659:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176284.876664:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176284.876670:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.876674:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176284.876677:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176284.876680:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176284.876684:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176284.876686:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.876688:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176284.876689:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176284.876693:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176284.876698:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176284.876705:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176284.876716:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235473408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176284.876746:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176284.876753:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176284.876763:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235473408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176284.876776:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176284.876780:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176284.876784:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176284.876786:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176284.876788:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176284.876791:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176284.876795:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10959 00000400:00000200:3.0:1786176284.876814:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176284.876822:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176284.876825:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176284.876830:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876834:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176284.876835:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876838:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176284.876841:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176284.876844:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176284.876846:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176284.876847:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176284.876850:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176284.876855:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176284.876858:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876860:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176284.876864:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.876870:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176284.876872:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176284.876877:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:2.1:1786176284.876882:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176284.876883:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176284.876887:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1786176284.876889:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 02000000:00000001:3.0:1786176284.876891:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176284.876896:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.876900:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176284.876902:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176284.876906:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.876908:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176284.876914:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176284.876918:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176284.876920:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176284.876922:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.876925:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176284.876929:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176284.876931:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176284.876936:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176284.876941:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176284.876948:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235473536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176284.876958:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176284.876963:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176284.876970:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235473536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176284.876978:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176284.876982:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176284.876986:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176284.876989:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176284.876992:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176284.876995:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176284.876998:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10959 00000400:00000200:3.0:1786176284.877002:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176284.877005:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176284.877008:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176284.877011:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.877013:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176284.877017:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.877021:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176284.877032:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176284.877035:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176284.877038:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176284.877040:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176284.877043:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176284.877045:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176284.877051:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.877055:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176284.877059:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.877062:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176284.877068:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176284.877074:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176284.877080:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176284.877084:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176284.877091:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176284.877096:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.877102:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176284.877107:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176284.877115:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.877119:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176284.877124:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176284.877129:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176284.877134:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176284.877137:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.877141:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176284.877143:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176284.877163:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176284.877168:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176284.877174:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176284.877182:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235473664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176284.877198:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176284.877202:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176284.877208:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235473664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176284.877222:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176284.877225:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176284.877230:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176284.877233:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176284.877235:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176284.877241:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176284.877245:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10959 00000400:00000200:3.0:1786176284.877253:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176284.877259:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176284.877263:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176284.877268:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.877273:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176284.877277:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.877282:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176284.877287:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176284.877290:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176284.877293:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176284.877296:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176284.877300:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176284.877303:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176284.877309:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.877313:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176284.877317:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176284.877320:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176284.877324:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176284.877328:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176284.877333:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176284.877335:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176284.877340:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176284.877341:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.877343:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176284.877344:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176284.877348:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.877351:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176284.877379:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176284.877384:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176284.877389:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176284.877391:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176284.877394:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176284.877397:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176284.877400:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176284.877405:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176284.877411:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176284.877421:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235473792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176284.877441:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176284.877443:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176284.877446:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235473792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176284.877459:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176284.877463:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10969) 00000800:00000200:2.1:1786176284.877536:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176284.877539:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176284.877773:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176284.877775:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176284.877986:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176284.877988:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176284.878288:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176284.878289:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000020:00000001:3.0:1786176285.772689:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176285.772695:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176285.772699:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176286.796573:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176286.796577:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176286.796580:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176287.820531:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176287.820534:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176287.820537:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176288.460525:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176288.460531:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176288.460534:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176288.460539:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176288.460543:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176288.460547:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176288.460549:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176288.460552:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176288.460556:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176288.844568:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176288.844600:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176288.844603:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176289.868562:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176289.868565:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176289.868568:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176289.997006:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176289.997014:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176289.997713:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176289.997714:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176289.997948:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176289.997950:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176289.998235:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176289.998238:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176289.998472:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176289.998474:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000020:00000001:3.0:1786176290.892576:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176290.892582:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176290.892586:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1786176291.916626:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176291.916631:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176291.916635:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176292.557173:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176292.557186:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176292.557189:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176292.557196:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176292.557203:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176292.557208:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176292.557212:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176292.557217:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176292.557222:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176292.940696:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176292.940701:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176292.940705:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176293.580542:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176293.580555:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176293.580558:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176293.580567:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176293.580574:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176293.580582:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176293.580587:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176293.580607:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176293.580612:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176293.964678:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176293.964683:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176293.964688:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176294.604463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176294.604473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176294.604486:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176294.604492:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176294.604498:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176294.604503:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176294.604506:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176294.604510:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176294.604526:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176294.988535:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176294.988540:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176294.988543:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176295.117023:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176295.117031:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176295.117631:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176295.117632:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176295.117820:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176295.117821:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176295.118122:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176295.118124:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176295.118326:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176295.118328:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1786176295.628487:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176295.628515:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176295.628519:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176295.628526:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176295.628532:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176295.628537:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176295.628541:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176295.628544:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176295.628549:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176296.012703:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176296.012709:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176296.012714:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176296.652460:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176296.652469:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176296.652472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176296.652479:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176296.652484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176296.652488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176296.652493:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176296.652496:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176296.652500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176297.036609:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176297.036612:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176297.036615:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176297.676443:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176297.676451:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176297.676454:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176297.676460:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176297.676465:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176297.676468:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176297.676470:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176297.676472:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176297.676474:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176298.060548:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176298.060552:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176298.060555:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176298.700472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176298.700485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176298.700505:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176298.700512:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176298.700518:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176298.700523:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176298.700528:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176298.700531:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176298.700537:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176299.084540:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176299.084544:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176299.084547:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176299.724535:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176299.724546:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176299.724549:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176299.724556:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176299.724562:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176299.724567:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176299.724571:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176299.724574:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176299.724588:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176300.108787:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176300.108793:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176300.108797:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176300.237622:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176300.237631:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786176300.238689:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176300.238695:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786176300.239212:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176300.239216:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786176300.239627:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176300.239630:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786176300.239984:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176300.239987:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1786176300.748465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176300.748475:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176300.748478:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176300.748484:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176300.748490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176300.748495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176300.748498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176300.748501:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176300.748505:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176301.132531:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176301.132534:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176301.132537:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176301.772514:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176301.772526:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176301.772529:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176301.772537:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176301.772544:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176301.772549:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176301.772553:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176301.772557:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176301.772561:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176302.156617:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176302.156623:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176302.156627:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176303.820459:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176303.820466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176303.820468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176303.820473:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176303.820476:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176303.820480:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176303.820482:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176303.820486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176303.820488:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176304.204527:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176304.204530:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176304.204533:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176304.844419:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176304.844426:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176304.844428:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176304.844433:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176304.844437:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176304.844440:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176304.844442:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176304.844445:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176304.844447:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176305.356482:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176305.356491:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176305.356493:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176305.356496:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176305.356501:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176305.356507:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 10979 00000400:00000200:3.0:1786176305.356515:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176305.356521:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176305.356524:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176305.356527:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.356529:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176305.356531:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.356533:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176305.356536:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176305.356548:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176305.356550:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176305.356552:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176305.356554:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176305.356555:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176305.356557:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.356559:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176305.356560:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.356562:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176305.356564:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176305.356567:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176305.356569:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176305.356570:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176305.356573:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176305.356575:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.356577:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176305.356579:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176305.356588:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786176305.356591:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176305.356594:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176305.356595:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.356597:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176305.356599:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176305.356615:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176305.356621:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176305.356626:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176305.356634:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235475840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176305.356643:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176305.356646:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176305.356650:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235475840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176305.356681:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176305.356683:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176305.356686:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176305.356687:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176305.356688:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176305.356690:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176305.356691:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10979 00000400:00000200:3.0:1786176305.356694:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176305.356696:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176305.356697:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176305.356700:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.356702:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176305.356704:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.356706:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176305.356707:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176305.356708:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176305.356710:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176305.356711:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176305.356713:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176305.356715:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176305.356717:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.356719:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176305.356723:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.356725:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176305.356729:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176305.356732:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176305.356735:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176305.356737:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176305.356742:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176305.356744:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.356748:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176305.356753:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176305.356759:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.356764:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176305.356773:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786176305.356778:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176305.356784:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176305.356789:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.356794:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176305.356798:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176305.356803:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176305.356817:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176305.356826:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176305.356839:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235475968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176305.356852:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176305.356858:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176305.356868:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235475968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176305.356896:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176305.356902:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176305.356910:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176305.356914:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176305.356919:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176305.356925:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176305.356933:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10979 00000400:00000200:3.0:1786176305.356978:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176305.356987:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176305.356990:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176305.356997:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357000:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176305.357006:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357008:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176305.357012:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176305.357013:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176305.357014:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176305.357017:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176305.357019:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176305.357023:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176305.357027:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357028:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176305.357029:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:2.1:1786176305.357032:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176305.357033:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176305.357038:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176305.357041:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357043:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176305.357044:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176305.357047:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176305.357048:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.357050:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176305.357051:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176305.357054:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.357057:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176305.357064:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786176305.357069:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176305.357072:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176305.357074:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.357075:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176305.357076:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176305.357078:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176305.357082:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176305.357086:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176305.357092:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235476096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176305.357101:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176305.357103:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176305.357109:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235476096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176305.357121:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176305.357126:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176305.357129:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176305.357134:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176305.357136:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176305.357142:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176305.357145:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10979 00000400:00000200:3.0:1786176305.357151:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176305.357154:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176305.357159:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176305.357162:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357164:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176305.357166:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357171:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176305.357177:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176305.357179:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176305.357183:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176305.357186:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176305.357190:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176305.357194:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176305.357198:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357201:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176305.357204:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357210:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176305.357215:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176305.357221:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357224:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176305.357228:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176305.357231:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176305.357234:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.357237:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176305.357241:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176305.357245:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.357248:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176305.357253:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176305.357257:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176305.357261:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176305.357263:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.357267:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176305.357270:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176305.357275:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176305.357278:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176305.357283:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176305.357290:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235476224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176305.357302:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176305.357306:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176305.357315:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235476224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176305.357327:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176305.357332:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176305.357336:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176305.357337:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176305.357339:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176305.357341:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176305.357344:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 10979 00000400:00000200:3.0:1786176305.357348:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176305.357380:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176305.357383:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176305.357386:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357387:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176305.357388:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357390:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176305.357395:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176305.357398:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176305.357402:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176305.357407:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176305.357413:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176305.357419:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176305.357428:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357434:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176305.357441:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357447:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176305.357453:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176305.357461:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176305.357469:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176305.357475:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176305.357482:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176305.357490:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.357495:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176305.357500:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176305.357505:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.357510:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176305.357516:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176305.357520:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176305.357525:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176305.357528:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176305.357531:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176305.357535:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176305.357539:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176305.357543:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000010:3.0:1786176305.357547:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176305.357554:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176305.357564:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235476352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176305.357584:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176305.357590:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176305.357600:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235476352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176305.357619:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176305.357627:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (10989) 00000800:00000200:2.1:1786176305.357774:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176305.357983:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176305.358161:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176305.868433:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176305.868440:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176305.868442:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176305.868447:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176305.868451:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176305.868454:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176305.868457:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176305.868459:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176305.868462:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176306.252546:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176306.252551:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176306.252555:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176306.892457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176306.892481:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176306.892483:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176306.892488:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176306.892492:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176306.892495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176306.892498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176306.892501:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176306.892504:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176307.916457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176307.916464:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176307.916466:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176307.916471:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176307.916474:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176307.916477:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176307.916479:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176307.916481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176307.916483:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176308.300572:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176308.300576:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176308.300579:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176308.940497:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176308.940506:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176308.940508:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176308.940530:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176308.940536:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176308.940541:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176308.940544:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176308.940548:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176308.940552:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176309.964450:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176309.964461:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176309.964464:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176309.964471:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176309.964475:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176309.964480:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176309.964483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176309.964489:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176309.964493:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176310.348676:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176310.348684:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176310.348691:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176310.477609:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176310.477620:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1786176310.478763:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176310.478766:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1786176310.479256:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176310.479259:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1786176310.479637:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176310.479639:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1786176310.479992:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176310.479994:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000100:00000001:3.0:1786176310.988505:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176310.988516:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176310.988520:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176310.988528:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176310.988533:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176310.988550:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176310.988553:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176310.988556:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176310.988560:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176312.012519:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176312.012528:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176312.012531:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176312.012537:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176312.012544:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176312.012548:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176312.012552:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176312.012555:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176312.012559:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176312.396535:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176312.396539:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176312.396542:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176313.036485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176313.036495:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176313.036498:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176313.036505:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176313.036510:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176313.036514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176313.036517:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176313.036520:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176313.036522:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176314.060467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176314.060476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176314.060478:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176314.060484:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176314.060491:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176314.060496:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176314.060500:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176314.060519:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176314.060522:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176314.444502:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176314.444506:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176314.444509:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176315.084508:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176315.084521:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176315.084527:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176315.084536:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176315.084543:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176315.084549:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176315.084554:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176315.084558:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176315.084563:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176315.596957:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176315.596965:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1786176315.597586:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176315.597588:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1786176315.597774:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176315.597776:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1786176315.598002:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176315.598004:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1786176315.598188:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176315.598189:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000100:00000001:3.0:1786176316.108434:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176316.108442:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176316.108445:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176316.108450:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176316.108454:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176316.108459:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176316.108462:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176316.108464:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176316.108467:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176316.492539:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176316.492543:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176316.492546:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176317.132443:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176317.132450:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176317.132452:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176317.132457:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176317.132461:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176317.132465:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176317.132467:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176317.132470:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176317.132473:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176318.156501:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176318.156514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176318.156517:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176318.156526:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176318.156533:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176318.156539:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176318.156543:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176318.156547:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176318.156551:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176318.540677:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176318.540685:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176318.540691:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176319.180493:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176319.180503:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176319.180505:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176319.180512:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176319.180518:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176319.180524:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176319.180528:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176319.180533:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176319.180538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176320.204446:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176320.204454:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176320.204457:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176320.204463:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176320.204467:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176320.204471:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176320.204474:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176320.204476:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176320.204479:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176320.588679:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176320.588685:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176320.588700:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176320.717193:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176320.717199:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176320.717887:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176320.717891:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176320.718233:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176320.718236:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176320.718540:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176320.718542:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176320.718999:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176320.719004:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1786176321.228490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176321.228500:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176321.228503:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176321.228510:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176321.228514:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176321.228519:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176321.228522:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176321.228525:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176321.228529:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176322.252491:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176322.252503:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176322.252508:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176322.252517:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176322.252523:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176322.252529:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176322.252533:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176322.252537:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176322.252542:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176322.636685:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176322.636691:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176322.636696:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176323.276513:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176323.276525:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176323.276528:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176323.276536:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176323.276542:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176323.276548:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176323.276552:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176323.276556:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176323.276561:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176324.300480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176324.300489:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176324.300492:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176324.300498:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176324.300504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176324.300508:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176324.300512:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176324.300515:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176324.300517:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176324.684553:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176324.684556:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176324.684559:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176325.324560:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176325.324574:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176325.324578:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176325.324586:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176325.324595:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176325.324603:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176325.324607:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176325.324629:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176325.324634:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176325.837075:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176325.837081:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786176325.837821:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176325.837824:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786176325.838056:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176325.838058:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786176325.838283:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176325.838286:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786176325.838602:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176325.838604:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1786176326.348418:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176326.348425:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176326.348427:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176326.348431:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176326.348435:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176326.348438:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176326.348440:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176326.348442:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176326.348445:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176326.732689:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176326.732694:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176326.732698:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176327.372433:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176327.372442:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176327.372460:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176327.372466:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176327.372470:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176327.372474:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176327.372476:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176327.372480:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176327.372483:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176328.396489:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176328.396501:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176328.396505:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176328.396512:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176328.396518:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176328.396523:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176328.396527:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176328.396532:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176328.396536:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176328.780468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176328.780470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176328.780473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176328.780474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176328.780545:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176328.780550:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176328.780555:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176329.420434:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176329.420442:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176329.420444:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176329.420448:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176329.420452:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176329.420455:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176329.420457:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176329.420460:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176329.420463:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176329.740479:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305676544, at 4305676608 00000100:00000010:0.0:1786176329.740500:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c720. 00010000:00000001:0.0:1786176329.804448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176329.804455:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176329.804460:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176329.804464:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176329.804471:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176329.804475:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176329.804478:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176329.804480:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176329.804484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176329.804486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176329.804489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176329.804491:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176329.804492:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176329.804494:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176329.804495:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176329.804496:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176329.804497:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176329.804498:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176330.444579:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176330.444588:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176330.444591:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176330.444597:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176330.444602:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176330.444607:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176330.444611:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176330.444614:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176330.444618:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176330.828478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176330.828488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176330.828495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176330.828499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176330.828507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176330.828512:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176330.828621:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176330.828628:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176330.828633:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176330.957534:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176330.957544:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176330.958909:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176330.958916:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176330.959463:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176330.959467:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176330.959959:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176330.959962:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176330.960328:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176330.960331:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1786176331.468452:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176331.468462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176331.468466:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176331.468472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176331.468479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176331.468484:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176331.468488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176331.468491:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176331.468496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176331.852453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176331.852460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176331.852471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176331.852475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176331.852479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176331.852481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176332.876535:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176332.876548:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176332.876563:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176332.876572:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176332.876583:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176332.876586:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176332.876749:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176332.876757:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176332.876763:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176333.516608:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176333.516617:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176333.516621:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176333.516629:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176333.516635:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176333.516640:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176333.516644:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176333.516647:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176333.516652:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176333.900468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176333.900479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176333.900492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176333.900496:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176333.900503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176333.900507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176334.540504:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176334.540514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176334.540518:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176334.540526:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176334.540533:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176334.540537:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176334.540541:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176334.540545:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176334.540549:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176334.860475:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305681665, at 4305681728 00000100:00000010:0.0:1786176334.860496:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cd80. 00010000:00000001:0.0:1786176334.924479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176334.924487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176334.924494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176334.924496:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176334.924502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176334.924504:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176334.924648:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176334.924654:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176334.924659:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176335.564443:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176335.564451:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176335.564453:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176335.564458:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176335.564466:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176335.564469:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176335.564472:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176335.564475:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176335.564477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176335.948428:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176335.948434:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176335.948440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176335.948443:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176335.948450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176335.948452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176336.076978:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176336.076984:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176336.077638:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176336.077640:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176336.077857:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176336.077858:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176336.078034:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176336.078036:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176336.078290:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176336.078293:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1786176336.588547:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176336.588561:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176336.588564:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176336.588573:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176336.588580:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176336.588585:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176336.588589:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176336.588594:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176336.588599:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176336.972552:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176336.972562:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176336.972574:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176336.972579:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176336.972583:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176336.972589:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176336.972594:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176336.972598:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176336.972602:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176336.972607:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176336.972613:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176336.972616:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176336.972756:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176336.972764:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176336.972771:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176337.612476:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176337.612490:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176337.612493:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176337.612500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176337.612505:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176337.612511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176337.612515:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176337.612518:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176337.612522:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176337.996510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176337.996519:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176337.996534:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176337.996541:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176337.996554:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176337.996561:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176338.636513:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176338.636526:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176338.636531:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176338.636540:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176338.636547:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176338.636554:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176338.636558:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176338.636563:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176338.636568:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176339.020428:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176339.020433:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176339.020439:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176339.020443:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176339.020449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176339.020452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176339.020548:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176339.020555:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176339.020560:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176339.660588:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176339.660601:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176339.660605:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176339.660613:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176339.660637:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176339.660642:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176339.660646:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176339.660651:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176339.660655:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176339.980464:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305686784, at 4305686848 00000100:00000010:0.0:1786176339.980485:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62f00. 00010000:00000001:0.0:1786176340.044434:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176340.044453:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176340.044456:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176340.044459:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176340.044464:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176340.044467:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176340.044469:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176340.044471:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176340.044478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176340.044480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176340.044485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176340.044490:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176340.044504:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176340.044513:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176340.044518:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176340.044524:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176340.044528:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176340.044534:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176340.684513:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176340.684524:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176340.684527:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176340.684535:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176340.684541:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176340.684546:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176340.684550:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176340.684554:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176340.684558:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176341.068468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176341.068474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176341.068482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176341.068486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176341.068496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176341.068501:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176341.068584:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176341.068589:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176341.068592:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176341.196949:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176341.196954:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176341.197578:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176341.197580:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176341.197788:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176341.197790:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176341.197964:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176341.197966:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176341.198209:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176341.198211:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1786176341.708441:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176341.708453:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176341.708456:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176341.708462:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176341.708468:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176341.708474:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176341.708477:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176341.708481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176341.708485:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176342.092470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176342.092476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176342.092493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176342.092497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176342.092501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176342.092504:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176342.732494:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176342.732504:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176342.732517:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176342.732523:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176342.732527:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176342.732531:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176342.732534:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176342.732537:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176342.732540:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176343.116535:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176343.116555:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176343.116563:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176343.116566:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176343.116576:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176343.116579:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176343.116661:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176343.116666:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176343.116670:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176343.756494:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176343.756502:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176343.756505:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176343.756512:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176343.756516:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176343.756519:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176343.756522:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176343.756525:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176343.756529:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176344.140472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176344.140481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176344.140495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176344.140502:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176344.140507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176344.140510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176344.780436:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176344.780445:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176344.780447:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176344.780452:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176344.780456:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176344.780460:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176344.780463:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176344.780466:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176344.780469:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176345.100454:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305691904, at 4305691968 00000100:00000010:0.0:1786176345.100468:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290ccc0. 00010000:00000001:0.0:1786176345.164442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176345.164446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176345.164452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176345.164454:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176345.164459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176345.164462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176345.164503:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176345.164507:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176345.164510:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176345.804423:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176345.804431:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176345.804434:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176345.804439:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176345.804443:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176345.804448:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176345.804452:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176345.804456:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176345.804460:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176346.188424:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176346.188429:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176346.188436:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176346.188439:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176346.188444:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176346.188445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176346.316488:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176346.316496:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176346.316498:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176346.316501:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176346.316505:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176346.316511:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11020 00000400:00000200:0.0:1786176346.316518:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176346.316523:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176346.316525:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176346.316527:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316530:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176346.316532:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316534:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176346.316538:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176346.316539:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176346.316541:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176346.316543:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176346.316545:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176346.316547:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176346.316548:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316550:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176346.316551:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316576:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176346.316579:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176346.316581:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316583:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176346.316585:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176346.316587:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176346.316589:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.316591:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176346.316593:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176346.316600:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176346.316603:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176346.316605:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176346.316606:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.316609:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176346.316612:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176346.316615:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176346.316623:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176346.316628:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176346.316635:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235480960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176346.316643:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176346.316646:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176346.316650:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235480960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176346.316667:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176346.316670:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176346.316672:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176346.316673:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176346.316674:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176346.316677:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176346.316678:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11020 00000400:00000200:0.0:1786176346.316681:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176346.316683:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176346.316685:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176346.316687:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316689:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176346.316690:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316692:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176346.316694:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176346.316696:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176346.316697:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176346.316698:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176346.316699:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176346.316702:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176346.316704:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316707:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176346.316711:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316713:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176346.316714:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176346.316717:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316719:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176346.316720:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176346.316724:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176346.316726:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.316730:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176346.316736:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176346.316740:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.316743:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176346.316751:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176346.316754:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176346.316760:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176346.316762:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.316767:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176346.316770:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176346.316774:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176346.316782:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176346.316790:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176346.316802:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235481088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176346.316828:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176346.316836:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176346.316849:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235481088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176346.316878:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176346.316881:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176346.316893:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176346.316900:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176346.316903:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176346.316906:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176346.316915:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11020 00000400:00000200:0.0:1786176346.316922:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176346.316925:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176346.316927:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176346.316935:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316938:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176346.316940:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.316953:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176346.316959:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176346.316970:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176346.316973:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176346.316975:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176346.316979:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176346.316983:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176346.316994:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317002:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176346.317007:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317015:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176346.317025:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176346.317029:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317037:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176346.317044:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176346.317048:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176346.317053:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.317057:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176346.317059:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176346.317062:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.317070:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176346.317075:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176346.317078:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176346.317079:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176346.317081:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.317087:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176346.317093:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176346.317095:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176346.317101:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176346.317107:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176346.317116:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235481216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176346.317131:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176346.317133:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176346.317139:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235481216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176346.317155:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176346.317158:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176346.317164:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176346.317168:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176346.317170:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176346.317181:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176346.317188:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11020 00000400:00000200:0.0:1786176346.317192:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176346.317194:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176346.317196:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176346.317198:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317199:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176346.317202:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317208:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176346.317216:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176346.317223:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176346.317227:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176346.317233:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176346.317236:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176346.317238:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176346.317242:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317249:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176346.317252:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317253:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176346.317262:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176346.317275:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317289:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176346.317296:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1786176346.317300:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:0.0:1786176346.317302:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1786176346.317309:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 02000000:00000001:0.0:1786176346.317311:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.317320:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176346.317326:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176346.317336:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.317344:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176346.317354:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176346.317397:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176346.317399:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176346.317400:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.317402:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176346.317403:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176346.317405:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176346.317409:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176346.317418:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176346.317425:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235481344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176346.317475:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176346.317481:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176346.317497:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235481344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176346.317513:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176346.317517:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176346.317522:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176346.317527:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176346.317531:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176346.317541:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176346.317548:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11020 00000400:00000200:0.0:1786176346.317571:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176346.317575:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176346.317577:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176346.317580:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317583:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176346.317588:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317596:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176346.317600:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176346.317603:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176346.317605:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176346.317608:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176346.317611:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176346.317616:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176346.317621:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317628:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176346.317631:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317634:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176346.317645:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176346.317658:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176346.317667:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176346.317670:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176346.317675:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176346.317684:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.317690:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176346.317694:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176346.317703:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.317712:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176346.317722:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176346.317727:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176346.317732:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176346.317734:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176346.317738:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176346.317745:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176346.317752:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176346.317762:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176346.317767:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176346.317784:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235481472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176346.317822:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176346.317832:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176346.317851:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235481472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176346.317871:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176346.317882:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11030) 00000800:00000200:2.1:1786176346.318095:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176346.318097:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176346.318415:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176346.318417:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176346.318636:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176346.318639:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176346.318826:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176346.318828:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1786176346.828448:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176346.828455:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176346.828458:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176346.828464:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176346.828469:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176346.828473:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176346.828476:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176346.828479:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176346.828483:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176347.212502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176347.212511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176347.212528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176347.212538:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176347.212693:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176347.212698:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176347.212702:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176347.212704:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176347.212706:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176347.212709:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176347.212713:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176347.212714:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176347.212762:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176347.212765:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176347.212767:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176347.852429:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176347.852436:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176347.852438:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176347.852443:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176347.852464:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176347.852468:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176347.852471:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176347.852474:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176347.852477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176348.236440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176348.236448:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176348.236457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176348.236461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176348.236475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176348.236479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176348.876432:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176348.876441:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176348.876444:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176348.876449:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176348.876454:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176348.876459:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176348.876462:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176348.876465:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176348.876469:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176349.260455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176349.260464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176349.260475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176349.260481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176349.260491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176349.260496:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176349.260557:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176349.260568:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176349.260571:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176349.900407:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176349.900416:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176349.900419:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176349.900425:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176349.900431:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176349.900437:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176349.900440:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176349.900460:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176349.900465:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176350.220499:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305697025, at 4305697088 00000100:00000010:0.0:1786176350.220543:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c960. 00010000:00000001:0.0:1786176350.284510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176350.284550:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176350.284552:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176350.284556:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176350.284559:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176350.284562:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176350.284564:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176350.284582:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176350.284586:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176350.284588:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176350.284590:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176350.284591:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176350.284593:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176350.284594:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176350.284596:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176350.284597:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176350.284598:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176350.284600:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176350.924497:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176350.924507:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176350.924539:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176350.924546:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176350.924552:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176350.924557:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176350.924560:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176350.924564:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176350.924567:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176351.308466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176351.308472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176351.308480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176351.308483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176351.308489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176351.308492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176351.308572:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176351.308576:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176351.308580:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176351.437534:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176351.437555:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1786176351.438227:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176351.438230:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1786176351.438600:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176351.438603:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1786176351.438906:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176351.438909:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 16 secs later) 00000800:00000200:1.1:1786176351.439199:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176351.439202:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 16 secs later) 00000100:00000001:3.0:1786176351.948456:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176351.948464:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176351.948466:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176351.948470:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176351.948474:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176351.948476:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176351.948478:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176351.948481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176351.948483:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176352.332465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176352.332469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176352.332479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176352.332483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176352.332488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176352.332491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1786176352.908916:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1786176352.972554:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176352.972566:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176352.972570:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176352.972580:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176352.972590:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176352.972622:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176352.972626:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176352.972630:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176352.972634:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176353.356586:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176353.356593:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176353.356606:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176353.356611:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176353.356618:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176353.356623:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176353.356719:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176353.356726:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176353.356731:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176353.996608:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176353.996619:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176353.996622:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176353.996631:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176353.996636:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176353.996641:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176353.996644:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176353.996647:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176353.996652:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176354.380516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176354.380521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176354.380530:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176354.380534:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176354.380539:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176354.380543:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176355.021159:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176355.021184:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176355.021189:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176355.021201:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176355.021210:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176355.021217:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176355.021223:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176355.021229:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176355.021234:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176355.340554:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305702145, at 4305702208 00000100:00000010:0.0:1786176355.340568:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62780. 00010000:00000001:0.0:1786176355.404502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176355.404509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176355.404518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176355.404523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176355.404531:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176355.404537:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176355.404681:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176355.404687:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176355.404692:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176356.044698:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176356.044716:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176356.044722:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176356.044734:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176356.044744:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176356.044754:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176356.044760:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176356.044767:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176356.044773:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176356.428517:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176356.428525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176356.428532:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176356.428534:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176356.428537:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176356.428538:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176356.428667:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176356.428675:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176356.428683:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176356.557359:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176356.557417:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786176356.558508:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176356.558511:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786176356.558835:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176356.558838:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786176356.559259:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176356.559262:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786176356.559695:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176356.559698:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000100:00000001:3.0:1786176357.068730:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176357.068747:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176357.068751:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176357.068763:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176357.068770:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176357.068779:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176357.068785:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176357.068790:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176357.068797:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176357.452478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176357.452486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176357.452500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176357.452505:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176357.452510:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176357.452517:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176357.452523:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176357.452526:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176357.452531:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176357.452545:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176357.452551:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176357.452557:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176357.452660:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176357.452667:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176357.452673:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176358.476544:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176358.476549:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176358.476557:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176358.476562:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176358.476568:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176358.476573:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176358.476657:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176358.476662:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176358.476666:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176359.500537:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176359.500548:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176359.500555:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176359.500563:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176359.500571:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176359.500576:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176359.500579:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176359.500582:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176359.500587:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176359.500589:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176359.500592:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176359.500594:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176359.500595:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176359.500597:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176359.500599:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176359.500600:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176359.500602:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176359.500605:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176359.500681:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176359.500686:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176359.500689:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176360.140559:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176360.140568:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176360.140572:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176360.140579:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176360.140586:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176360.140592:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176360.140597:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176360.140601:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176360.140606:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176360.460489:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305707265, at 4305707328 00000100:00000010:0.0:1786176360.460508:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62480. 00010000:00000001:0.0:1786176360.524474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176360.524481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176360.524488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176360.524498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176360.524505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176360.524509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176360.524580:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176360.524583:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176360.524586:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176361.164566:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176361.164590:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176361.164593:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176361.164599:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176361.164603:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176361.164607:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176361.164611:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176361.164613:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176361.164617:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176361.548541:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176361.548549:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176361.548559:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176361.548566:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176361.548574:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176361.548576:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176361.548656:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176361.548660:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176361.548663:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176361.677122:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176361.677128:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176361.677914:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176361.677916:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176361.678215:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176361.678217:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176361.678566:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176361.678568:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176361.678815:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176361.678817:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000100:00000001:0.0:1786176362.188492:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176362.188502:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176362.188506:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176362.188513:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176362.188520:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176362.188525:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176362.188529:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176362.188533:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176362.188536:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176362.572453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176362.572459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176362.572466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176362.572471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176362.572480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176362.572481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176362.572537:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176362.572541:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176362.572544:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176363.212459:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176363.212466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176363.212468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176363.212472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176363.212476:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176363.212479:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176363.212481:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176363.212483:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176363.212486:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176363.596457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176363.596462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176363.596468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176363.596471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176363.596478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176363.596480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176363.596535:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176363.596539:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176363.596541:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176364.236472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176364.236482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176364.236484:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176364.236489:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176364.236493:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176364.236497:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176364.236500:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176364.236503:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176364.236506:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176364.620567:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176364.620573:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176364.620583:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176364.620587:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176364.620593:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176364.620597:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176364.620652:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176364.620656:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176364.620659:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176365.260581:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176365.260590:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176365.260592:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176365.260598:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176365.260603:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176365.260607:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176365.260611:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176365.260615:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176365.260619:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176365.580478:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305712384, at 4305712448 00000100:00000010:0.0:1786176365.580495:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62d20. 00010000:00000001:0.0:1786176365.644470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176365.644474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176365.644482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176365.644487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176365.644492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176365.644495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176365.644600:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176365.644606:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176365.644610:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176366.284538:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176366.284549:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176366.284554:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176366.284563:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176366.284569:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176366.284574:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176366.284578:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176366.284582:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176366.284588:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176366.668461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176366.668466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176366.668475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176366.668479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176366.668485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176366.668489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176366.668553:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176366.668556:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176366.668559:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176366.797045:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176366.797050:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786176366.797451:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176366.797452:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786176366.797668:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176366.797670:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786176366.798009:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176366.798012:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786176366.798257:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176366.798259:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00010000:00000001:0.0:1786176367.692506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176367.692513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176367.692522:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176367.692526:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176367.692528:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176367.692532:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176367.692541:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176367.692544:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176367.692549:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176367.692552:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176367.692558:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176367.692562:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176367.692641:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176367.692646:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176367.692650:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176368.716524:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176368.716532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176368.716552:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176368.716558:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176368.716596:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176368.716599:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176368.716671:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176368.716677:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176368.716680:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176369.740436:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176369.740442:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176369.740445:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176369.740449:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176369.740453:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176369.740456:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176369.740460:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176369.740464:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176369.740472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176369.740473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176369.740476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176369.740477:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176369.740478:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176369.740479:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176369.740480:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176369.740481:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176369.740482:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176369.740484:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176369.740549:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176369.740553:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176369.740555:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176370.700480:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305717504, at 4305717568 00000100:00000010:0.0:1786176370.700506:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62240. 00010000:00000001:0.0:1786176370.764487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176370.764492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176370.764497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176370.764502:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176370.764505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176370.764509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176370.764576:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176370.764582:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176370.764586:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176371.788434:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176371.788440:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176371.788445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176371.788447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176371.788450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176371.788452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176371.788517:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176371.788521:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176371.788524:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176371.917638:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176371.918585:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176371.919089:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176371.919502:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176371.919791:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00010000:00000001:0.0:1786176372.812460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176372.812466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176372.812475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176372.812479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176372.812484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176372.812487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176372.812607:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176372.812614:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176372.812618:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176373.836542:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176373.836553:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176373.836561:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176373.836565:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176373.836573:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176373.836575:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176373.836684:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176373.836694:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176373.836702:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176374.476529:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176374.476543:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176374.476548:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176374.476556:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176374.476561:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176374.476567:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176374.476571:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176374.476575:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176374.476581:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176374.860495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176374.860506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176374.860516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176374.860522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176374.860535:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176374.860539:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176374.860681:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176374.860691:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176374.860698:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1786176374.988625:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:0.0:1786176374.988664:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1786176374.988673:0:2383:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176374.988677:0:2383:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:0.0:1786176374.988681:0:2383:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176374.988684:0:2383:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1786176374.988689:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176374.988693:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:0.0:1786176374.988698:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176374.988702:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176374.988708:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ec38 00000400:00000010:0.0:1786176374.988710:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ec38. 00000100:00000001:0.0:1786176374.988716:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176374.988727:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31734a80 x1872930235484160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176396 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176374.988778:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176374.988785:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734a80 x1872930235484160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176396 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176374.988796:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176374.988799:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176374.988812:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176374.988815:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:0.0:1786176374.988819:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176374.988824:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176374.988829:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e7f8 00000400:00000010:0.0:1786176374.988832:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e7f8. 00000100:00000001:0.0:1786176374.988835:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176374.988841:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735f80 x1872930235484288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176396 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176374.988853:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176374.988857:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735f80 x1872930235484288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176396 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176374.988868:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176374.988870:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176374.988876:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176374.988880:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:0.0:1786176374.988900:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176374.988904:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176374.988908:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e880 00000400:00000010:0.0:1786176374.988911:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e880. 00000100:00000001:0.0:1786176374.988913:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176374.988918:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31737100 x1872930235484416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176396 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176374.988927:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176374.988940:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235484416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176396 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176374.988954:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176374.988956:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176374.988962:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176374.988964:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:0.0:1786176374.988968:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176374.988971:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176374.988975:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e110 00000400:00000010:0.0:1786176374.988977:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e110. 00000100:00000001:0.0:1786176374.988980:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176374.988985:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31736a00 x1872930235484544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176396 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176374.988994:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176374.988998:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235484544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176396 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176374.989007:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176374.989009:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176374.989054:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176374.989058:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:0.0:1786176374.989062:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176374.989067:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176374.989073:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ea18 00000400:00000010:0.0:1786176374.989076:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ea18. 00000100:00000001:0.0:1786176374.989080:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176374.989087:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31736680 x1872930235484672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176396 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176374.989098:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176374.989105:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235484672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176396 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176374.989119:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176374.989123:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786176375.500471:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176375.500480:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176375.500483:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176375.500489:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176375.500494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176375.500498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176375.500501:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176375.500504:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176375.500507:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176375.820473:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305722625, at 4305722688 00000100:00000010:0.0:1786176375.820493:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ae0. 00010000:00000001:0.0:1786176375.884511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176375.884519:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176375.884530:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176375.884536:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176375.884546:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176375.884553:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176375.884692:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176375.884699:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176375.884704:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176376.524470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176376.524484:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176376.524487:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176376.524496:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176376.524501:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176376.524506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176376.524510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176376.524514:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176376.524529:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176376.908503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176376.908510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176376.908516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176376.908517:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176376.908521:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176376.908522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176376.908590:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176376.908596:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176376.908600:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176377.036525:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176377.036535:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176377.036538:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176377.036542:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176377.036546:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176377.036553:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11051 00000400:00000200:3.0:1786176377.036561:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176377.036567:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176377.036570:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176377.036573:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.036576:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176377.036578:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.036580:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176377.036583:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176377.036585:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176377.036588:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176377.036591:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176377.036593:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176377.036595:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176377.036597:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.036599:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176377.036600:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.036602:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176377.036604:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176377.036606:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176377.036608:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176377.036610:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176377.036614:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176377.036616:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.036618:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176377.036631:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176377.036645:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176377.036648:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176377.036652:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176377.036653:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.036656:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176377.036658:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176377.036663:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176377.036670:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176377.036676:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176377.036684:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235484800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176377.036693:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176377.036698:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176377.036705:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235484800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176377.036733:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176377.036738:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176377.036743:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176377.036744:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176377.036746:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176377.036750:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176377.036753:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11051 00000400:00000200:3.0:1786176377.036758:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176377.036762:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176377.036765:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176377.036769:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.036772:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176377.036776:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.036780:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176377.036784:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176377.036786:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176377.036791:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176377.036797:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176377.036803:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176377.036807:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176377.036817:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.036824:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176377.036830:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.036835:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176377.036842:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176377.036850:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176377.036856:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176377.036862:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176377.036871:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176377.036880:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.036884:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176377.036891:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176377.036897:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.036903:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176377.036911:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176377.036915:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176377.036922:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176377.036926:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.036930:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176377.036933:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176377.036938:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176377.036945:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176377.036952:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176377.036966:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235484928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176377.036987:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176377.036995:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176377.037007:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235484928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176377.037032:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176377.037039:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176377.037046:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176377.037048:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176377.037053:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176377.037061:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176377.037065:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11051 00000400:00000200:3.0:1786176377.037076:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176377.037086:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176377.037091:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176377.037100:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.037106:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176377.037114:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.037121:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176377.037128:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176377.037133:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176377.037139:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176377.037145:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176377.037153:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176377.037159:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176377.037164:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.037167:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176377.037169:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.037172:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176377.037175:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176377.037183:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176377.037191:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176377.037198:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176377.037205:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176377.037212:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.037217:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176377.037220:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176377.037227:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.037229:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176377.037235:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176377.037242:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176377.037246:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176377.037250:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.037254:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176377.037276:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176377.037278:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176377.037284:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176377.037288:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176377.037296:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00100000:3.0:1786176377.037297:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235485056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1786176377.037306:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786176377.037330:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176377.037336:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176377.037344:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235485056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176377.037402:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176377.037406:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176377.037411:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176377.037417:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176377.037419:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176377.037424:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176377.037432:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11051 00000400:00000200:3.0:1786176377.037443:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176377.037817:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176377.037827:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176377.037836:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.037841:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176377.037849:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.037856:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176377.037863:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176377.037867:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176377.037872:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176377.037877:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176377.037883:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176377.037889:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176377.037899:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.037905:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176377.037912:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.037919:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176377.037927:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176377.037939:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176377.037947:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176377.037953:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176377.037961:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176377.037970:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.037975:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176377.037981:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176377.037988:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.037993:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176377.038000:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176377.038008:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176377.038017:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176377.038021:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.038027:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176377.038032:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176377.038037:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176377.038047:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000800:00000200:1.1:1786176377.038054:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176377.038055:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176377.038061:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00100000:3.0:1786176377.038070:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235485184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176377.038084:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176377.038090:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176377.038101:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235485184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176377.038126:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176377.038133:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176377.038143:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176377.038147:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176377.038533:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176377.038541:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176377.038551:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11051 00000400:00000200:3.0:1786176377.038565:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:1.1:1786176377.038572:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:3.0:1786176377.038577:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000200:1.1:1786176377.038579:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00080000:3.0:1786176377.038581:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176377.038589:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.038593:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176377.038599:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.038603:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176377.038609:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176377.038611:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176377.038614:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176377.038617:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176377.038621:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176377.038624:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176377.038631:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.038635:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176377.038639:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176377.038646:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176377.038649:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176377.038655:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176377.038657:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176377.038661:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176377.038666:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176377.038674:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.038679:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176377.038682:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176377.038687:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.038690:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176377.038694:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176377.038699:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176377.038702:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176377.038705:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176377.038709:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176377.038711:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176377.038716:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176377.038724:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176377.038733:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176377.038745:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235485312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176377.038771:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176377.038778:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176377.038792:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235485312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176377.038830:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176377.038839:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11061) 00000800:00000200:1.1:1786176377.039100:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176377.039102:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176377.039416:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176377.039419:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1786176377.548526:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176377.548537:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176377.548540:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176377.548548:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176377.548554:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176377.548559:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176377.548563:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176377.548566:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176377.548570:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176377.932520:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176377.932528:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176377.932538:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176377.932544:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176377.932549:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176377.932555:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176377.932559:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176377.932562:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176377.932564:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176377.932567:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176377.932570:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176377.932571:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176377.932642:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176377.932647:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176377.932652:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176378.572550:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176378.572561:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176378.572564:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176378.572571:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176378.572577:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176378.572583:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176378.572587:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176378.572590:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176378.572596:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176378.956493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176378.956503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176378.956516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176378.956522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176378.956533:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176378.956539:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176378.956625:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176378.956630:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176378.956635:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176379.596505:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176379.596515:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176379.596517:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176379.596525:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176379.596544:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176379.596549:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176379.596552:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176379.596556:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176379.596560:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176379.980541:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176379.980553:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176379.980558:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176379.980564:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176379.980594:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176379.980600:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176379.980606:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176379.980613:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176379.980623:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176379.980629:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176379.980636:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176379.980642:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176379.980644:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176379.980647:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176379.980649:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176379.980651:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176379.980653:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176379.980655:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176379.980764:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176379.980772:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176379.980778:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176380.940519:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305727745, at 4305727808 00000100:00000010:0.0:1786176380.940536:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62d20. 00010000:00000001:0.0:1786176381.004477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176381.004481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176381.004487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176381.004490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176381.004495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176381.004497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176381.004602:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176381.004618:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176381.004622:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176382.028452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176382.028459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176382.028471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176382.028476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176382.028484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176382.028489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176382.028577:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176382.028581:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176382.028584:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176382.157592:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176382.157598:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176382.158873:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176382.158877:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176382.159256:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176382.159259:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176382.159669:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176382.159672:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176382.160124:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176382.160127:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1786176383.052446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176383.052452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176383.052459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176383.052462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176383.052467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176383.052469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176383.052545:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176383.052550:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176383.052553:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176384.076440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176384.076445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176384.076454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176384.076457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176384.076461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176384.076465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176384.076527:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176384.076531:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176384.076534:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176384.716523:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176384.716531:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176384.716533:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176384.716538:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176384.716542:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176384.716545:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176384.716547:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176384.716550:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176384.716553:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176385.100523:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176385.100545:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176385.100554:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176385.100560:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176385.100567:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176385.100584:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176385.100689:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176385.100694:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176385.100698:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176385.740504:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176385.740517:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176385.740522:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176385.740530:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176385.740537:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176385.740544:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176385.740550:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176385.740555:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176385.740562:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176386.060523:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305732865, at 4305732928 00000100:00000010:0.0:1786176386.060558:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c420. 00010000:00000001:0.0:1786176386.124452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176386.124456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176386.124463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176386.124466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176386.124471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176386.124474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176386.124579:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176386.124585:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176386.124589:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176386.764478:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176386.764489:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176386.764492:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176386.764500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176386.764507:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176386.764513:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176386.764516:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176386.764520:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176386.764525:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176387.148506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176387.148515:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176387.148530:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176387.148535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176387.148540:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176387.148542:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176387.148642:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176387.148648:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176387.148652:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176387.277318:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176387.277328:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176387.278432:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176387.278434:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176387.278746:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176387.278749:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176387.279064:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176387.279066:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176387.279382:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176387.279384:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1786176387.788523:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176387.788536:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176387.788541:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176387.788550:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176387.788556:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176387.788562:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176387.788566:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176387.788571:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176387.788576:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176388.172515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176388.172521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176388.172526:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176388.172529:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176388.172531:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176388.172535:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176388.172538:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176388.172540:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176388.172542:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176388.172543:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176388.172546:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176388.172547:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176388.172618:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176388.172623:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176388.172626:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176388.812433:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176388.812440:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176388.812442:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176388.812447:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176388.812450:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176388.812453:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176388.812456:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176388.812458:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176388.812460:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176389.196523:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176389.196531:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176389.196538:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176389.196540:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176389.196543:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176389.196545:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176389.196657:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176389.196667:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176389.196673:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176389.836454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176389.836466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176389.836469:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176389.836476:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176389.836482:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176389.836488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176389.836492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176389.836511:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176389.836516:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176390.220499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176390.220507:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176390.220509:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176390.220513:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176390.220518:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176390.220522:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176390.220524:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176390.220527:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176390.220533:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176390.220535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176390.220539:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176390.220541:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176390.220543:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176390.220545:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176390.220547:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176390.220548:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176390.220559:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176390.220563:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176390.220672:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176390.220676:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176390.220679:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.0:1786176390.796502:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:3.0:1786176390.796510:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:3.0:1786176390.796516:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1786176390.796732:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1786176390.796740:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176390.796753:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786176390.796759:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176390.796762:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176390.860495:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176390.860506:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176390.860508:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176390.860514:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176390.860521:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176390.860527:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176390.860531:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176390.860533:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176390.860536:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176391.180443:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305737985, at 4305738048 00000100:00000010:0.0:1786176391.180456:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62480. 00010000:00000001:0.0:1786176391.244435:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176391.244441:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176391.244446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176391.244449:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176391.244452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176391.244455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176391.244498:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176391.244502:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176391.244505:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176391.884433:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176391.884440:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176391.884442:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176391.884447:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176391.884450:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176391.884453:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176391.884455:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176391.884457:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176391.884459:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176392.268428:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176392.268434:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176392.268439:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176392.268441:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176392.268443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176392.268445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176392.268497:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176392.268502:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176392.268506:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176392.397063:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176392.397070:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176392.397656:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176392.397659:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176392.397981:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176392.397983:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176392.398308:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176392.398311:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176392.398637:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176392.398640:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1786176392.908446:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176392.908455:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176392.908458:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176392.908463:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176392.908468:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176392.908471:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176392.908474:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176392.908477:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176392.908480:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176393.292466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176393.292471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176393.292479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176393.292481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176393.292485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176393.292487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176393.292544:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176393.292548:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176393.292552:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176394.316425:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176394.316430:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176394.316435:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176394.316438:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176394.316443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176394.316446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176394.316496:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176394.316499:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176394.316502:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176395.340459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176395.340463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176395.340469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176395.340472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176395.340476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176395.340478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176395.340535:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176395.340540:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176395.340543:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176396.300427:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305743104, at 4305743168 00000100:00000010:0.0:1786176396.300442:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62780. 00010000:00000001:0.0:1786176396.364470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176396.364474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176396.364481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176396.364484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176396.364488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176396.364491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176396.364549:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176396.364552:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176396.364555:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176397.388466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176397.388473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176397.388481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176397.388485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176397.388494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176397.388497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176397.388562:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176397.388565:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176397.388569:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176397.516985:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176397.516990:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176397.517681:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176397.517682:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176397.517866:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176397.517868:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176397.518041:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176397.518042:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176397.518309:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176397.518310:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00010000:00000001:0.0:1786176398.412441:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176398.412445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176398.412453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176398.412458:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176398.412461:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176398.412465:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176398.412470:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176398.412474:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176398.412477:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176398.412480:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176398.412484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176398.412486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176398.412574:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176398.412578:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176398.412581:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176399.436519:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176399.436530:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176399.436534:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176399.436547:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176399.436560:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176399.436568:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176399.436582:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176399.436589:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176399.436595:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176399.436599:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176399.436602:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176399.436605:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176399.436606:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176399.436608:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176399.436609:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176399.436610:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176399.436611:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176399.436613:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176399.436645:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176399.436647:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176399.436649:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176400.460448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176400.460462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176400.460470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176400.460474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176400.460481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176400.460485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176400.460542:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176400.460546:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176400.460549:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176401.420424:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305748224, at 4305748288 00000100:00000010:0.0:1786176401.420438:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c360. 00010000:00000001:0.0:1786176401.484454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176401.484458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176401.484466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176401.484468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176401.484471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176401.484474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176401.484495:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176401.484498:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176401.484501:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176402.508438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176402.508442:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176402.508450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176402.508452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176402.508456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176402.508458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176402.508523:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176402.508526:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176402.508529:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176402.636933:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176402.636938:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176402.637597:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176402.637602:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176402.637798:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176402.637800:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176402.637975:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176402.637976:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176402.638233:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176402.638235:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00010000:00000001:0.0:1786176403.532429:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176403.532434:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176403.532441:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176403.532444:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176403.532448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176403.532451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176403.532470:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176403.532473:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176403.532475:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176404.556438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176404.556444:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176404.556454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176404.556459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176404.556464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176404.556469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176404.556524:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176404.556528:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176404.556531:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176405.580484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176405.580493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176405.580505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176405.580510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176405.580516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176405.580519:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176405.580546:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176405.580549:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176405.580552:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176406.540455:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305753344, at 4305753408 00000100:00000010:0.0:1786176406.540475:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62f00. 00010000:00000001:0.0:1786176406.604448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176406.604453:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176406.604459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176406.604463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176406.604466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176406.604468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176406.604528:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176406.604533:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176406.604537:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176407.628452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176407.628458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176407.628463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176407.628467:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176407.628470:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176407.628473:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176407.628478:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176407.628482:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176407.628487:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176407.628489:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176407.628492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176407.628493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176407.628513:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176407.628515:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176407.628518:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176407.757043:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176407.757050:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176407.757808:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176407.757810:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176407.758023:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176407.758024:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176407.758207:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176407.758209:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176407.758509:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176407.758511:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00010000:00000001:0.0:1786176408.652442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176408.652446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176408.652454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176408.652457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176408.652460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176408.652463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176408.652513:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176408.652517:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176408.652520:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176409.676448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176409.676454:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176409.676458:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176409.676462:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176409.676468:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176409.676472:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176409.676477:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176409.676482:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176409.676489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176409.676492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176409.676496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176409.676497:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176409.676498:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176409.676500:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176409.676502:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176409.676504:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176409.676506:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176409.676508:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176409.676530:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176409.676534:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176409.676537:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176410.700450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176410.700458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176410.700464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176410.700467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176410.700470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176410.700472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176411.660449:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305758464, at 4305758528 00000100:00000010:0.0:1786176411.660465:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62360. 00010000:00000001:0.0:1786176411.724430:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176411.724434:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176411.724440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176411.724441:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176411.724445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176411.724447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176411.724467:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176411.724470:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176411.724473:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176412.748458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176412.748464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176412.748473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176412.748478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176412.748488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176412.748491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176412.877094:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176412.877101:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176412.877940:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176412.877943:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176412.878267:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176412.878270:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176412.878608:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176412.878611:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176412.879056:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176412.879058:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1786176413.772439:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176413.772445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176413.772451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176413.772454:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176413.772461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176413.772465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176413.772483:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176413.772486:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176413.772488:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176414.796439:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176414.796451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176414.796457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176414.796459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176414.796462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176414.796463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176415.820429:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176415.820435:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176415.820442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176415.820445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176415.820450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176415.820451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176415.820473:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176415.820476:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176415.820479:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176416.780427:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305763584, at 4305763648 00000100:00000010:0.0:1786176416.780446:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c600. 00010000:00000001:0.0:1786176416.844430:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176416.844434:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176416.844440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176416.844442:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176416.844447:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176416.844450:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176417.868460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176417.868467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176417.868473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176417.868476:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176417.868479:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176417.868482:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176417.868485:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176417.868488:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176417.868490:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176417.868492:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176417.868496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176417.868498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176417.868522:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176417.868525:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176417.868528:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176417.996946:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176417.996953:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176417.997507:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176417.997509:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176417.997692:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176417.997693:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176417.997916:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176417.997918:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176417.998132:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176417.998134:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1786176418.892482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176418.892489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176418.892500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176418.892516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176418.892521:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176418.892523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176419.916469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176419.916496:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176419.916500:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176419.916504:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176419.916508:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176419.916512:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176419.916516:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176419.916518:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176419.916524:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176419.916528:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176419.916533:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176419.916536:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176419.916537:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176419.916539:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176419.916540:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176419.916541:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176419.916543:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176419.916545:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176419.916566:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176419.916568:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176419.916570:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176420.940471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176420.940475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176420.940482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176420.940486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176420.940490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176420.940493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176421.900467:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305768704, at 4305768768 00000100:00000010:0.0:1786176421.900485:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c480. 00010000:00000001:0.0:1786176421.964440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176421.964445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176421.964450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176421.964452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176421.964457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176421.964458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176421.964481:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176421.964484:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176421.964487:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176422.988462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176422.988467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176422.988475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176422.988479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176422.988485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176422.988489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176423.117092:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176423.117097:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176423.117809:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176423.117839:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176423.118212:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176423.118214:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176423.118504:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176423.118506:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176423.118682:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176423.118683:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1786176424.012447:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176424.012451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176424.012457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176424.012460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176424.012464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176424.012468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176424.012496:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176424.012500:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176424.012502:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176425.036439:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176425.036444:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176425.036449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176425.036452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176425.036457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176425.036460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176426.060513:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176426.060523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176426.060536:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176426.060541:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176426.060548:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176426.060552:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176426.060582:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176426.060588:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176426.060594:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176427.020514:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305773824, at 4305773888 00000100:00000010:0.0:1786176427.020533:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c1e0. 00010000:00000001:0.0:1786176427.084435:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176427.084439:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176427.084445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176427.084446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176427.084450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176427.084451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176428.108504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176428.108511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176428.108520:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176428.108527:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176428.108532:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176428.108540:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176428.108552:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176428.108557:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176428.108559:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176428.108562:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176428.108566:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176428.108568:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176428.108593:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176428.108596:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176428.108599:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176428.237637:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176428.237648:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176428.238848:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176428.238852:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176428.239268:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176428.239273:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176428.239727:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176428.239731:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176428.240138:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176428.240141:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.0:1786176428.684873:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:0.0:1786176429.132499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176429.132505:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176429.132512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176429.132517:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176429.132529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176429.132531:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176430.156464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176430.156476:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176430.156479:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176430.156484:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176430.156492:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176430.156495:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176430.156498:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176430.156502:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176430.156508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176430.156510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176430.156516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176430.156519:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176430.156523:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176430.156527:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176430.156533:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176430.156541:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176430.156551:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176430.156554:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176430.156588:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176430.156594:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176430.156598:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176431.180557:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176431.180568:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176431.180584:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176431.180588:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176431.180615:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176431.180619:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176431.820579:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176431.820590:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176431.820592:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176431.820599:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176431.820604:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176431.820609:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176431.820629:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176431.820635:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176431.820641:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176432.140554:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305778945, at 4305779008 00000100:00000010:0.0:1786176432.140569:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c240. 00010000:00000001:0.0:1786176432.204541:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176432.204547:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176432.204560:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176432.204564:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176432.204574:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176432.204578:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176432.204609:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176432.204613:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176432.204618:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176432.844489:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176432.844499:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176432.844503:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176432.844511:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176432.844516:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176432.844522:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176432.844527:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176432.844531:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176432.844535:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176433.228443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176433.228447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176433.228454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176433.228457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176433.228461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176433.228463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176433.356612:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176433.356624:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176433.356628:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176433.356645:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176433.356651:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176433.356660:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11107 00000400:00000200:3.0:1786176433.356669:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176433.356676:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176433.356680:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176433.356685:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.356689:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176433.356692:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.356695:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176433.356699:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176433.356702:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176433.356705:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176433.356708:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176433.356711:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176433.356753:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176433.356757:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.356759:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176433.356761:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.356764:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176433.356768:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176433.356772:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176433.356774:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176433.356778:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176433.356781:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176433.356784:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.356787:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176433.356790:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176433.356801:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176433.356805:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176433.356810:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176433.356812:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.356815:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176433.356818:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176433.356823:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176433.356831:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176433.356839:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176433.356849:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235491840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176433.356862:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176433.356868:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176433.356873:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235491840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176433.356901:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176433.356905:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176433.356909:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176433.356911:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176433.356913:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176433.356917:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176433.356919:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11107 00000400:00000200:3.0:1786176433.356924:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176433.356929:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176433.356932:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176433.356937:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.356940:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176433.356943:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.356947:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176433.356950:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176433.356952:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176433.356954:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176433.356956:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176433.356959:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176433.356966:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176433.356969:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.356971:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176433.356975:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.356978:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176433.356982:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176433.356985:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176433.356987:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176433.356990:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176433.356997:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176433.357002:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.357008:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176433.357017:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176433.357031:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.357041:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176433.357053:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176433.357064:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176433.357076:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176433.357082:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.357090:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176433.357098:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176433.357108:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176433.357123:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176433.357133:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176433.357149:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235491968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176433.357169:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176433.357177:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176433.357196:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235491968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176433.357244:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176433.357250:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176433.357260:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176433.357266:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176433.357270:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176433.357274:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176433.357286:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11107 00000400:00000200:3.0:1786176433.357350:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176433.357399:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176433.357403:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176433.357408:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.357410:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176433.357414:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.357417:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176433.357420:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176433.357422:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176433.357426:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176433.357431:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176433.357436:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176433.357444:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176433.357454:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.357460:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176433.357464:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.357474:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176433.357477:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176433.357484:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176433.357490:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176433.357495:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176433.357504:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176433.357509:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.357516:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176433.357520:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176433.357527:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176433.357529:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176433.357531:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176433.357541:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000800:00000200:2.1:1786176433.357542:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000020:00000040:3.0:1786176433.357548:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176433.357555:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176433.357560:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.357564:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176433.357569:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176433.357575:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176433.357583:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176433.357594:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176433.357607:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235492096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176433.357626:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176433.357632:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176433.357643:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235492096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176433.357697:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176433.357701:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176433.357709:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176433.357734:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176433.357736:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176433.357741:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176433.357752:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11107 00000400:00000200:3.0:1786176433.357765:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176433.357777:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176433.357782:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176433.357787:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.357796:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176433.357804:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.357810:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176433.357818:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176433.357819:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176433.357823:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176433.357829:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176433.357833:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176433.357837:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176433.357845:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.357848:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176433.357853:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.357859:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176433.357866:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176433.357877:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176433.357887:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176433.357892:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176433.357899:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176433.357907:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.357912:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176433.357918:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176433.357925:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.357932:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176433.357941:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786176433.357947:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176433.357955:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176433.357959:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.357965:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176433.357974:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176433.357981:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176433.357989:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176433.357998:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176433.358013:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235492224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176433.358047:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176433.358057:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176433.358077:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235492224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176433.358139:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176433.358147:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176433.358158:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176433.358164:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176433.358169:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176433.358175:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176433.358186:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11107 00000400:00000200:3.0:1786176433.358199:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176433.358210:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176433.358218:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176433.358229:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.358237:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176433.358247:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.358252:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176433.358258:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176433.358262:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176433.358269:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176433.358272:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176433.358275:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176433.358285:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176433.358292:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.358296:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176433.358304:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176433.358311:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176433.358318:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176433.358327:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176433.358335:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176433.358340:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176433.358349:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176433.358355:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.358395:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176433.358401:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176433.358407:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.358413:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176433.358425:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786176433.358432:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176433.358442:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176433.358447:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176433.358451:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176433.358453:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176433.358456:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176433.358463:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176433.358471:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176433.358482:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235492352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176433.358498:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176433.358505:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176433.358519:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235492352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176433.358544:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176433.358556:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11117) 00000800:00000200:2.1:1786176433.358709:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176433.358733:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786176433.359060:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176433.359063:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786176433.359434:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176433.359437:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786176433.359932:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176433.359936:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000001:0.0:1786176433.868466:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176433.868473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176433.868475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176433.868480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176433.868484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176433.868488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176433.868490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176433.868492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176433.868495:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176434.252456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176434.252460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176434.252468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176434.252472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176434.252476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176434.252478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176434.892442:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176434.892448:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176434.892450:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176434.892455:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176434.892458:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176434.892462:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176434.892464:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176434.892466:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176434.892468:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176435.276447:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176435.276453:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176435.276460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176435.276463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176435.276466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176435.276467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176435.916429:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176435.916442:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176435.916444:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176435.916449:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176435.916453:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176435.916456:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176435.916458:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176435.916460:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176435.916463:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176436.300456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176436.300463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176436.300471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176436.300475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176436.300480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176436.300482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176436.940459:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176436.940468:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176436.940471:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176436.940477:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176436.940480:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176436.940483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176436.940485:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176436.940487:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176436.940490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176437.260478:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305784065, at 4305784128 00000100:00000010:0.0:1786176437.260491:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cba0. 00010000:00000001:0.0:1786176437.324476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176437.324483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176437.324492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176437.324501:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176437.324519:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176437.324531:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176437.964445:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176437.964453:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176437.964456:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176437.964463:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176437.964467:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176437.964471:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176437.964474:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176437.964477:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176437.964480:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176438.348482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176438.348491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176438.348501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176438.348506:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176438.348528:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176438.348532:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176438.348538:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176438.348543:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176438.348558:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176438.348563:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176438.348567:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176438.348568:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176438.476423:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176438.476433:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176438.476435:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176438.476438:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176438.476441:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176438.476447:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11112 00000400:00000200:3.0:1786176438.476454:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176438.476458:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176438.476461:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176438.476464:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476466:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176438.476469:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476470:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176438.476475:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176438.476477:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176438.476480:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176438.476484:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176438.476487:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176438.476490:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176438.476493:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476495:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176438.476497:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476499:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176438.476502:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176438.476505:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476508:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176438.476511:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176438.476515:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176438.476518:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.476520:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176438.476522:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176438.476532:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786176438.476536:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176438.476540:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176438.476542:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.476545:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176438.476549:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176438.476553:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176438.476561:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176438.476568:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176438.476576:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235492480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176438.476585:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176438.476590:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176438.476595:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235492480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176438.476616:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176438.476619:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176438.476633:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176438.476635:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176438.476637:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176438.476640:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176438.476643:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11112 00000400:00000200:3.0:1786176438.476647:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176438.476653:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176438.476657:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176438.476696:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476702:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176438.476709:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476716:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176438.476721:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176438.476725:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176438.476727:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176438.476730:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176438.476732:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176438.476736:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176438.476742:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476744:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176438.476746:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476752:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176438.476756:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176438.476761:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476768:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176438.476771:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176438.476777:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176438.476781:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.476787:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176438.476791:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176438.476795:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.476798:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176438.476805:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176438.476809:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176438.476814:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176438.476816:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.476819:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176438.476822:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176438.476825:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176438.476831:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176438.476838:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176438.476849:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235492608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176438.476868:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176438.476875:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176438.476886:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235492608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176438.476910:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176438.476914:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176438.476920:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176438.476921:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176438.476924:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176438.476927:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176438.476932:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11112 00000800:00000200:2.1:1786176438.476934:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:3.0:1786176438.476939:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176438.476944:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176438.476946:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176438.476948:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476950:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176438.476951:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476952:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176438.476954:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176438.476955:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176438.476957:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176438.476961:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176438.476965:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176438.476971:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176438.476976:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476979:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176438.476984:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476987:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176438.476989:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176438.476993:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176438.476995:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176438.476997:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176438.477012:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176438.477015:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.477018:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176438.477022:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176438.477026:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.477029:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176438.477035:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176438.477039:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176438.477043:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176438.477044:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.477045:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176438.477049:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176438.477054:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176438.477060:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176438.477066:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176438.477074:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235492736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176438.477103:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176438.477107:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176438.477123:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235492736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176438.477138:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176438.477158:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176438.477162:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176438.477165:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176438.477170:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176438.477176:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176438.477181:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11112 00000400:00000200:3.0:1786176438.477187:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176438.477192:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176438.477197:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176438.477203:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.477206:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176438.477207:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.477216:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176438.477220:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176438.477223:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176438.477227:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176438.477230:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176438.477233:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176438.477237:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176438.477245:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.477250:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176438.477257:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.477261:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176438.477266:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176438.477273:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176438.477277:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176438.477278:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176438.477282:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176438.477286:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.477292:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176438.477297:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176438.477302:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.477306:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176438.477313:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786176438.477318:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176438.477324:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176438.477327:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.477329:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176438.477332:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176438.477334:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176438.477335:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000010:3.0:1786176438.477340:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176438.477344:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176438.477354:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235492864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176438.477410:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176438.477414:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176438.477422:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235492864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176438.477443:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176438.477448:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176438.477457:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176438.477463:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176438.477468:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176438.477475:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176438.477482:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11112 00000400:00000200:3.0:1786176438.477491:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176438.477498:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176438.477504:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176438.477514:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.477519:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176438.477527:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.477533:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176438.477537:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176438.477539:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176438.477545:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176438.477551:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176438.477557:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176438.477563:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176438.477571:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.477577:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176438.477584:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176438.477590:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176438.477595:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176438.477604:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176438.477612:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176438.477617:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176438.477622:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176438.477630:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.477635:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176438.477640:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176438.477647:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.477652:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176438.477660:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786176438.477681:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176438.477687:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176438.477691:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176438.477698:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176438.477702:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176438.477707:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176438.477715:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176438.477722:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176438.477733:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235492992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176438.477750:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176438.477755:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176438.477760:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786176438.477766:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235492992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176438.477778:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176438.477783:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11123) 00000800:00000200:2.1:1786176438.477947:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176438.478104:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786176438.988454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176438.988461:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176438.988462:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176438.988467:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176438.988471:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176438.988475:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176438.988477:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176438.988480:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176438.988482:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176439.372455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176439.372461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176439.372470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176439.372474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176439.372482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176439.372485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176440.012464:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176440.012472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176440.012474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176440.012480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176440.012484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176440.012488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176440.012490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176440.012493:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176440.012496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176440.396546:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176440.396556:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176440.396561:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176440.396567:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176440.396573:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176440.396576:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176440.396580:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176440.396586:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176440.396591:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176440.396594:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176440.396596:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176440.396598:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176440.396599:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176440.396601:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176440.396602:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176440.396604:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176440.396605:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176440.396606:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176441.036455:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176441.036466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176441.036469:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176441.036475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176441.036480:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176441.036485:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176441.036488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176441.036492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176441.036495:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176441.420443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176441.420449:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176441.420457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176441.420459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176441.420461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176441.420462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176442.060443:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176442.060451:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176442.060453:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176442.060458:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176442.060462:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176442.060466:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176442.060468:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176442.060472:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176442.060484:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176442.380466:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305789184, at 4305789248 00000100:00000010:0.0:1786176442.380481:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c6c0. 00010000:00000001:0.0:1786176442.444494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176442.444499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176442.444508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176442.444512:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176442.444521:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176442.444525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176443.084492:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176443.084501:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176443.084503:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176443.084509:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176443.084512:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176443.084516:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176443.084519:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176443.084521:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176443.084524:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176443.468499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176443.468506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176443.468511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176443.468513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176443.468516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176443.468519:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176443.596524:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176443.596540:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176443.596544:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176443.596549:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176443.596555:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176443.596565:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11118 00000400:00000200:3.0:1786176443.596576:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176443.596584:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176443.596588:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176443.596593:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.596597:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176443.596600:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.596604:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176443.596608:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176443.596611:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176443.596649:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176443.596653:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176443.596657:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176443.596660:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176443.596663:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.596666:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176443.596668:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.596670:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176443.596674:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176443.596678:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176443.596681:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176443.596685:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176443.596690:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176443.596694:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.596697:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176443.596700:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176443.596712:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176443.596717:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176443.596723:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176443.596725:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.596729:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176443.596733:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176443.596738:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176443.596748:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176443.596757:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176443.596776:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235493120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176443.596797:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176443.596803:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176443.596810:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235493120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176443.596844:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176443.596849:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176443.596853:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176443.596855:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176443.596857:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176443.596861:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176443.596863:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11118 00000400:00000200:3.0:1786176443.596868:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176443.596872:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176443.596876:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176443.596880:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.596884:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176443.596886:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.596889:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176443.596893:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176443.596894:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176443.596897:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176443.596900:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176443.596903:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176443.596906:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176443.596911:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.596914:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176443.596916:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.596918:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176443.596923:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176443.596941:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176443.596951:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176443.596958:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176443.596970:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176443.596979:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.596981:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176443.596990:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176443.597001:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.597010:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176443.597024:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176443.597030:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176443.597037:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176443.597042:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.597048:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176443.597053:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176443.597056:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176443.597069:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176443.597095:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176443.597120:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235493248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176443.597133:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176443.597141:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176443.597153:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235493248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176443.597186:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176443.597201:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176443.597209:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176443.597213:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176443.597214:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176443.597224:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176443.597229:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11118 00000400:00000200:3.0:1786176443.597239:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176443.597249:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176443.597260:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176443.597270:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.597279:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176443.597289:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.597301:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176443.597312:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176443.597318:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176443.597325:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176443.597331:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176443.597341:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176443.597351:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176443.597434:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.597441:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176443.597450:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.597460:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176443.597469:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176443.597481:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176443.597492:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176443.597547:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176443.597552:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 02000000:00000001:3.0:1786176443.597803:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176443.597813:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176443.597820:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.597831:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176443.597838:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176443.597844:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.597847:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176443.597854:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176443.597859:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176443.597866:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176443.597868:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.597871:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176443.597875:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176443.597879:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176443.597901:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176443.597911:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176443.597920:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235493376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176443.597931:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176443.597936:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176443.597941:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235493376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176443.597950:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176443.597954:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176443.597959:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176443.597961:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176443.597963:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176443.597966:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176443.597970:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11118 00000400:00000200:3.0:1786176443.597975:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176443.597980:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176443.597983:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176443.597987:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.597991:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176443.597993:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.597996:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176443.597999:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176443.598001:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176443.598004:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176443.598008:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176443.598011:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176443.598013:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176443.598017:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.598018:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176443.598022:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.598026:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176443.598029:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176443.598035:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176443.598043:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176443.598046:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176443.598051:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176443.598056:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.598074:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176443.598080:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176443.598086:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.598092:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176443.598101:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176443.598106:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176443.598112:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176443.598116:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.598118:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176443.598120:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176443.598125:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176443.598132:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176443.598139:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176443.598152:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235493504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176443.598172:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176443.598178:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176443.598186:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235493504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176443.598207:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176443.598212:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176443.598221:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176443.598224:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176443.598228:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176443.598234:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176443.598240:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11118 00000400:00000200:3.0:1786176443.598249:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176443.598256:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176443.598258:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176443.598266:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.598272:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176443.598280:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.598284:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176443.598290:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176443.598293:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176443.598297:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176443.598302:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176443.598306:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176443.598313:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176443.598322:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.598328:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176443.598335:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176443.598338:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176443.598341:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176443.598349:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176443.598354:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176443.598415:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176443.598420:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176443.598424:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.598428:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176443.598432:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176443.598439:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.598445:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176443.598468:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176443.598475:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176443.598486:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176443.598489:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176443.598494:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176443.598498:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176443.598502:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176443.598513:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176443.598520:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176443.598526:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00100000:3.0:1786176443.598531:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235493632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1786176443.598531:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786176443.598551:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176443.598556:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176443.598567:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235493632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176443.598584:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176443.598592:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11128) 00000800:00000200:2.1:1786176443.598856:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176443.598858:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176443.599099:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176443.599100:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176443.599274:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176443.599276:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1786176444.108468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176444.108474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176444.108476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176444.108480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176444.108483:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176444.108486:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176444.108488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176444.108491:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176444.108493:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176444.492462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176444.492467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176444.492474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176444.492478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176444.492486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176444.492487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176445.132452:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176445.132460:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176445.132462:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176445.132467:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176445.132471:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176445.132475:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176445.132477:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176445.132480:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176445.132484:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176445.516469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176445.516473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176445.516478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176445.516482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176445.516486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176445.516488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176446.540466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176446.540473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176446.540482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176446.540487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176446.540495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176446.540500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176447.500457:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305794305, at 4305794368 00000100:00000010:0.0:1786176447.500471:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c2a0. 00010000:00000001:0.0:1786176447.564427:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176447.564431:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176447.564437:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176447.564440:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176447.564443:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176447.564447:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176447.564450:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176447.564452:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176447.564454:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176447.564456:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176447.564460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176447.564462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176448.588443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176448.588447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176448.588454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176448.588455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176448.588458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176448.588459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176448.716492:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176448.716502:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176448.716505:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176448.716509:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176448.716514:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176448.716521:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11123 00000400:00000200:0.0:1786176448.716530:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176448.716537:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176448.716540:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176448.716544:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.716548:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176448.716551:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.716554:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176448.716558:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176448.716560:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176448.716578:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176448.716582:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176448.716585:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176448.716588:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176448.716591:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.716594:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176448.716596:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.716598:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176448.716601:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176448.716605:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176448.716624:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176448.716627:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176448.716630:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176448.716633:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.716636:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176448.716638:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176448.716648:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786176448.716652:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176448.716656:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176448.716658:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.716662:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176448.716665:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176448.716670:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176448.716678:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176448.716685:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176448.716697:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235493760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176448.716709:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176448.716714:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176448.716720:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235493760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176448.716744:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176448.716748:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176448.716752:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176448.716754:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176448.716756:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176448.716784:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176448.716788:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11123 00000400:00000200:0.0:1786176448.716794:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176448.716800:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176448.716803:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176448.716806:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.716809:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176448.716812:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.716815:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176448.716819:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176448.716823:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176448.716826:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176448.716833:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176448.716836:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176448.716842:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176448.716849:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.716853:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176448.716887:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.716893:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176448.716905:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176448.716914:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176448.716922:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176448.716926:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176448.716934:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176448.716942:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.716947:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176448.716955:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176448.716963:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.716967:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176448.716980:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786176448.716986:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176448.716993:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176448.716995:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.716997:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176448.716999:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176448.717002:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176448.717007:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176448.717031:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176448.717041:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235493888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176448.717053:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176448.717060:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176448.717066:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235493888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176448.717092:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176448.717099:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176448.717112:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176448.717116:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176448.717120:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176448.717126:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176448.717136:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11123 00000400:00000200:0.0:1786176448.717145:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176448.717154:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176448.717161:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176448.717171:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.717173:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176448.717180:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.717185:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176448.717188:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176448.717193:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176448.717198:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176448.717204:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176448.717209:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176448.717214:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176448.717222:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.717227:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176448.717237:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.717246:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176448.717263:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176448.717272:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176448.717280:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176448.717285:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176448.717292:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176448.717300:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.717309:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176448.717314:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176448.717321:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.717324:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176448.717333:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786176448.717339:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176448.717345:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:2.1:1786176448.717347:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:0.0:1786176448.717348:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176448.717358:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1786176448.717391:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176448.717395:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176448.717397:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176448.717403:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176448.717409:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176448.717421:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235494016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176448.717442:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176448.717450:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176448.717464:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235494016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176448.717496:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176448.717504:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176448.717511:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176448.717514:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176448.717519:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176448.717526:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176448.717532:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11123 00000400:00000200:0.0:1786176448.717546:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176448.717554:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176448.717560:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176448.717588:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.717596:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176448.717616:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.717621:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176448.717627:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176448.717631:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176448.717635:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176448.717641:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176448.717649:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176448.717653:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176448.717665:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.717669:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176448.717676:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.717680:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176448.717684:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176448.717694:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176448.717701:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176448.717709:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176448.717718:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176448.717725:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.717731:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176448.717737:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176448.717743:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.717751:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176448.717760:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786176448.717784:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176448.717791:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176448.717795:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.717799:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176448.717805:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176448.717808:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176448.717820:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176448.717830:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176448.717842:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235494144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176448.717875:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176448.717883:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176448.717901:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235494144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176448.717956:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176448.717961:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176448.717970:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176448.717978:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176448.717984:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176448.717989:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176448.717998:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11123 00000400:00000200:0.0:1786176448.718010:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176448.718019:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176448.718028:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176448.718034:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.718039:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176448.718048:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.718072:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176448.718078:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176448.718082:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176448.718088:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176448.718093:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176448.718100:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176448.718106:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176448.718115:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.718121:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176448.718126:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176448.718132:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176448.718140:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176448.718149:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176448.718156:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176448.718163:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176448.718168:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176448.718173:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.718182:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176448.718186:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176448.718192:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.718200:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176448.718206:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786176448.718213:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176448.718218:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176448.718223:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176448.718229:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176448.718233:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176448.718238:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176448.718245:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176448.718252:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176448.718266:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235494272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176448.718304:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176448.718311:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176448.718317:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235494272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176448.718339:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176448.718348:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11133) 00000800:00000200:2.1:1786176448.718556:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176448.718560:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176448.719061:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176448.719066:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176448.719377:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176448.719380:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176448.719978:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176448.719983:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1786176449.612533:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176449.612543:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176449.612550:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176449.612573:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176449.612580:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176449.612583:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176449.612586:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176449.612590:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176449.612608:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176449.612611:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176449.612617:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176449.612620:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176449.612622:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176449.612624:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176449.612626:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176449.612628:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176449.612630:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176449.612633:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176450.636590:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176450.636594:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176450.636598:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176450.636599:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176450.636602:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176450.636604:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176451.660452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176451.660458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176451.660463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176451.660465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176451.660469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176451.660471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176452.620514:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305799425, at 4305799488 00000100:00000010:0.0:1786176452.620548:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cf60. 00010000:00000001:0.0:1786176452.684475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176452.684481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176452.684489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176452.684494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176452.684502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176452.684506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176453.708462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176453.708467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176453.708472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176453.708475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176453.708480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176453.708482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176453.836972:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176453.836976:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176453.837657:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176453.837661:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176453.837851:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176453.837853:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176453.838079:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176453.838081:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176453.838330:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176453.838332:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00010000:00000001:0.0:1786176454.732434:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176454.732441:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176454.732448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176454.732451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176454.732455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176454.732457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176454.732523:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176454.732525:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176454.732527:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176455.756457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176455.756461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176455.756469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176455.756471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176455.756476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176455.756479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176456.780533:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176456.780540:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176456.780555:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176456.780558:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176456.780563:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176456.780568:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176457.740474:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305804544, at 4305804608 00000100:00000010:0.0:1786176457.740511:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c0c0. 00010000:00000001:0.0:1786176457.804509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176457.804514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176457.804521:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176457.804526:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176457.804530:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176457.804535:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176457.804542:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176457.804546:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176457.804551:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176457.804553:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176457.804559:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176457.804560:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176458.828490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176458.828497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176458.828504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176458.828506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176458.828509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176458.828510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176458.956540:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176458.956553:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176458.956557:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176458.956560:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176458.956566:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176458.956575:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11133 00000400:00000200:3.0:1786176458.956584:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176458.956591:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176458.956595:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176458.956599:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.956603:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176458.956606:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.956609:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176458.956613:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176458.956616:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176458.956619:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176458.956623:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176458.956626:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176458.956628:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176458.956632:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.956634:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176458.956636:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.956639:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176458.956642:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176458.956646:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176458.956649:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176458.956652:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176458.956656:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176458.956659:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.956662:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176458.956665:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176458.956672:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176458.956676:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176458.956680:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176458.956682:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.956685:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176458.956687:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176458.956691:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176458.956699:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176458.956706:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176458.956717:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235495040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176458.956729:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176458.956734:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176458.956739:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235495040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176458.956772:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176458.956776:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176458.956779:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176458.956781:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176458.956787:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176458.956791:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176458.956793:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11133 00000400:00000200:3.0:1786176458.956798:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176458.956801:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176458.956804:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176458.956808:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.956811:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176458.956814:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.956817:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176458.956819:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176458.956821:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176458.956826:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176458.956829:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176458.956834:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176458.956838:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176458.956845:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.956848:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176458.956853:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.956858:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176458.956864:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176458.956870:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176458.956877:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176458.956882:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176458.956899:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176458.956903:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.956907:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176458.956914:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176458.956919:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.956923:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176458.956929:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176458.956934:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176458.956939:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176458.956943:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.956946:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176458.956948:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176458.956951:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176458.956958:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176458.956966:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176458.956976:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235495168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176458.956999:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176458.957004:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176458.957014:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235495168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176458.957031:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176458.957036:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176458.957044:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176458.957048:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176458.957052:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176458.957058:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176458.957064:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11133 00000400:00000200:3.0:1786176458.957084:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176458.957093:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176458.957099:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176458.957108:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957114:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176458.957120:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957126:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:2.1:1786176458.957126:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176458.957132:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000020:00000001:3.0:1786176458.957133:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176458.957137:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176458.957141:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176458.957145:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176458.957149:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176458.957155:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176458.957162:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957167:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176458.957174:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957181:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176458.957186:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176458.957193:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957199:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176458.957204:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176458.957209:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176458.957215:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.957218:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176458.957223:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176458.957228:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.957233:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176458.957240:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176458.957244:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176458.957250:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176458.957253:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.957257:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176458.957261:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176458.957264:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176458.957272:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176458.957279:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176458.957290:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235495296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176458.957309:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176458.957314:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176458.957324:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235495296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176458.957344:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176458.957350:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176458.957388:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176458.957390:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176458.957392:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176458.957396:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176458.957403:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11133 00000400:00000200:3.0:1786176458.957413:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176458.957422:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176458.957428:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176458.957436:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957441:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176458.957448:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957453:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176458.957459:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176458.957462:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176458.957480:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176458.957485:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176458.957491:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176458.957497:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176458.957506:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957512:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176458.957518:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957524:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176458.957530:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176458.957538:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957545:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176458.957550:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176458.957555:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176458.957560:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.957565:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176458.957570:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176458.957576:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.957581:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176458.957588:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176458.957593:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176458.957599:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176458.957603:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.957608:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176458.957612:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176458.957618:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176458.957627:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176458.957634:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176458.957645:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235495424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176458.957662:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176458.957668:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176458.957677:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235495424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176458.957698:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176458.957703:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176458.957710:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176458.957714:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176458.957718:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176458.957724:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176458.957730:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11133 00000400:00000200:3.0:1786176458.957755:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176458.957766:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176458.957770:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176458.957778:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957783:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176458.957788:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957793:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176458.957799:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176458.957803:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176458.957806:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:2.1:1786176458.957808:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786176458.957810:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000800:00000200:2.1:1786176458.957812:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000020:00000001:3.0:1786176458.957815:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176458.957821:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176458.957829:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957834:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176458.957841:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957847:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176458.957853:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176458.957860:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176458.957865:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176458.957870:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176458.957874:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176458.957880:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.957884:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176458.957889:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176458.957892:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.957897:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176458.957905:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176458.957910:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176458.957915:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176458.957918:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176458.957921:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176458.957925:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176458.957928:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176458.957935:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176458.957942:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176458.957953:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235495552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176458.957971:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176458.957977:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176458.957987:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235495552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176458.958017:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176458.958024:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11143) 00000800:00000200:2.1:1786176458.958241:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176458.958243:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786176458.958652:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176458.958654:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786176458.958949:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176458.958951:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00010000:00000001:0.0:1786176459.852515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176459.852524:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176459.852528:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176459.852532:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176459.852537:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176459.852540:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176459.852542:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176459.852545:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176459.852551:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176459.852553:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176459.852558:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176459.852560:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176459.852562:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176459.852564:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176459.852566:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176459.852567:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176459.852569:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176459.852572:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176460.876491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176460.876497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176460.876502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176460.876503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176460.876505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176460.876506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176461.900457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176461.900461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176461.900466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176461.900469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176461.900472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176461.900475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176462.860453:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305809664, at 4305809728 00000100:00000010:0.0:1786176462.860473:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cf60. 00010000:00000001:0.0:1786176462.924454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176462.924459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176462.924465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176462.924468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176462.924471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176462.924473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176463.948473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176463.948479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176463.948485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176463.948487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176463.948491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176463.948493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176464.076484:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176464.076500:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176464.076502:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176464.076505:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176464.076508:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176464.076514:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11138 00000400:00000200:0.0:1786176464.076521:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176464.076526:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176464.076528:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176464.076531:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076534:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176464.076536:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076537:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176464.076540:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176464.076542:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176464.076544:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176464.076546:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176464.076548:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176464.076550:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176464.076552:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076553:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176464.076555:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076556:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176464.076558:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176464.076560:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076562:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176464.076564:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176464.076566:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176464.076568:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.076570:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176464.076571:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176464.076579:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786176464.076582:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176464.076584:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176464.076586:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.076588:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176464.076589:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176464.076593:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176464.076601:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176464.076606:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176464.076613:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235495680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176464.076627:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176464.076632:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176464.076638:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235495680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176464.076659:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176464.076662:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176464.076666:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176464.076668:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176464.076670:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176464.076673:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176464.076677:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11138 00000400:00000200:0.0:1786176464.076682:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176464.076686:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176464.076689:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176464.076694:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076700:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176464.076707:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076712:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176464.076718:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176464.076722:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176464.076727:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176464.076731:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176464.076739:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176464.076743:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176464.076752:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076755:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176464.076760:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076765:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176464.076770:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176464.076773:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076777:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176464.076779:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176464.076784:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176464.076787:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.076791:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176464.076794:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176464.076799:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.076800:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176464.076804:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786176464.076809:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176464.076813:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176464.076814:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.076816:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176464.076819:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176464.076822:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176464.076827:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176464.076833:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176464.076842:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235495808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176464.076856:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176464.076861:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176464.076870:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235495808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176464.076894:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176464.076900:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176464.076904:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176464.076907:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176464.076910:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176464.076913:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176464.076920:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11138 00000400:00000200:0.0:1786176464.076926:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176464.076931:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176464.076934:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176464.076941:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076944:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176464.076949:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076954:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:2.1:1786176464.076954:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:0.0:1786176464.076959:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:2.1:1786176464.076960:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000020:00000040:0.0:1786176464.076962:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176464.076965:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176464.076969:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176464.076974:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176464.076979:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176464.076982:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076985:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176464.076988:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076989:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176464.076991:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176464.076992:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176464.076994:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176464.076995:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176464.076997:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176464.077001:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.077005:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176464.077007:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176464.077009:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.077011:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176464.077017:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786176464.077022:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176464.077025:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176464.077027:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.077030:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176464.077031:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176464.077034:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176464.077038:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176464.077043:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176464.077052:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235495936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176464.077066:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176464.077070:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176464.077078:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235495936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176464.077086:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176464.077092:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176464.077098:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176464.077101:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176464.077104:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176464.077106:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176464.077109:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11138 00000400:00000200:0.0:1786176464.077114:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176464.077116:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176464.077121:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176464.077125:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.077128:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176464.077133:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.077136:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176464.077139:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176464.077142:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176464.077145:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176464.077148:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176464.077152:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176464.077157:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176464.077164:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.077165:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176464.077166:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.077167:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176464.077169:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176464.077171:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176464.077175:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176464.077177:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176464.077182:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176464.077185:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.077189:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176464.077192:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176464.077195:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.077198:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176464.077203:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786176464.077205:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176464.077210:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176464.077212:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.077215:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176464.077218:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176464.077221:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176464.077225:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176464.077229:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176464.077236:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235496064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176464.077248:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176464.077252:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176464.077260:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235496064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176464.077271:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176464.077276:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176464.077283:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176464.077286:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176464.077289:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176464.077293:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176464.077297:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11138 00000400:00000200:0.0:1786176464.077304:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176464.077309:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176464.077313:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176464.077319:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.077322:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176464.077326:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.077331:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176464.077335:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176464.077337:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176464.077340:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176464.077342:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176464.077346:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176464.077349:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176464.077374:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.077377:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176464.077379:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176464.077380:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176464.077382:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176464.077385:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176464.077388:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176464.077390:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176464.077393:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176464.077398:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.077402:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176464.077404:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176464.077409:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.077412:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176464.077432:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786176464.077435:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176464.077439:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176464.077441:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176464.077444:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176464.077447:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176464.077450:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176464.077456:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176464.077462:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176464.077468:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235496192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176464.077479:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176464.077482:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176464.077489:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235496192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176464.077497:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176464.077501:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11148) 00000800:00000200:2.1:1786176464.077606:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176464.077608:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176464.077800:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176464.077802:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176464.078019:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176464.078021:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176464.078242:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176464.078243:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00010000:00000001:0.0:1786176464.972489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176464.972493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176464.972516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176464.972520:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176464.972523:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176464.972526:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176465.996459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176465.996464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176465.996468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176465.996472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176465.996475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176465.996477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176467.020470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176467.020474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176467.020480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176467.020482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176467.020487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176467.020490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1786176467.596715:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1786176467.980468:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305814784, at 4305814848 00000100:00000010:0.0:1786176467.980486:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c2a0. 00010000:00000001:0.0:1786176468.044468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176468.044472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176468.044478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176468.044483:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176468.044486:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176468.044490:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176468.044495:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176468.044499:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176468.044503:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176468.044505:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176468.044509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176468.044511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176469.068443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176469.068451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176469.068459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176469.068462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176469.068473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176469.068479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176469.196464:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176469.196476:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176469.196478:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176469.196482:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176469.196487:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176469.196494:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11143 00000400:00000200:3.0:1786176469.196504:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176469.196514:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176469.196517:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176469.196521:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.196526:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176469.196529:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.196532:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176469.196537:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176469.196541:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176469.196544:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176469.196548:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176469.196551:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176469.196554:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176469.196558:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.196561:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176469.196563:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.196565:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176469.196568:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176469.196571:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176469.196573:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176469.196576:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176469.196579:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176469.196581:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.196583:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176469.196586:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176469.196597:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786176469.196601:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176469.196605:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176469.196607:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.196610:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176469.196613:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176469.196617:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176469.196625:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176469.196632:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176469.196642:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235496320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176469.196652:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176469.196656:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176469.196661:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235496320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176469.196689:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176469.196691:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176469.196694:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176469.196696:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176469.196697:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176469.196699:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176469.196701:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11143 00000400:00000200:3.0:1786176469.196704:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176469.196706:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176469.196708:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176469.196710:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.196712:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176469.196714:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.196716:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176469.196718:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176469.196719:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176469.196722:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176469.196725:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176469.196727:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176469.196730:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176469.196738:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.196744:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176469.196750:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.196756:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176469.196761:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176469.196768:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176469.196774:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176469.196780:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176469.196799:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176469.196805:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.196808:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176469.196814:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176469.196820:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.196824:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176469.196831:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786176469.196836:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176469.196839:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176469.196841:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.196853:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176469.196856:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176469.196860:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176469.196870:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176469.196877:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176469.196891:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235496448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176469.196907:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176469.196912:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176469.196921:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235496448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176469.196934:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176469.196939:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176469.196946:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176469.196951:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176469.196953:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176469.196957:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176469.196969:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11143 00000400:00000200:3.0:1786176469.196986:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176469.196993:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176469.196997:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176469.197004:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197007:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176469.197011:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197015:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176469.197017:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176469.197018:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176469.197021:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176469.197024:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176469.197025:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176469.197027:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176469.197033:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197036:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176469.197039:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197043:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176469.197046:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000200:2.1:1786176469.197047:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786176469.197052:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:2.1:1786176469.197053:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000100:00000001:3.0:1786176469.197060:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176469.197063:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176469.197068:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176469.197071:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.197074:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176469.197077:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176469.197081:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.197084:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176469.197087:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786176469.197091:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176469.197096:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176469.197098:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.197100:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176469.197102:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176469.197105:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176469.197114:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176469.197119:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176469.197125:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235496576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176469.197134:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176469.197138:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176469.197145:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235496576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176469.197153:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176469.197156:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176469.197161:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176469.197162:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176469.197164:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176469.197166:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176469.197169:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11143 00000400:00000200:3.0:1786176469.197178:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176469.197184:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176469.197188:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176469.197191:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197192:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176469.197193:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197194:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176469.197196:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176469.197197:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176469.197198:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176469.197200:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176469.197203:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176469.197206:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176469.197208:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197210:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176469.197212:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197216:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176469.197218:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176469.197221:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197227:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176469.197232:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176469.197235:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176469.197240:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.197242:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176469.197245:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176469.197249:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.197252:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176469.197258:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176469.197262:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176469.197267:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176469.197269:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.197274:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176469.197276:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176469.197277:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176469.197283:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176469.197286:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176469.197290:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235496704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176469.197299:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176469.197302:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176469.197306:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235496704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176469.197315:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176469.197317:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176469.197321:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176469.197323:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176469.197326:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176469.197330:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176469.197334:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11143 00000400:00000200:3.0:1786176469.197339:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176469.197344:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176469.197348:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176469.197353:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197398:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176469.197401:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197403:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176469.197407:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176469.197408:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176469.197412:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176469.197414:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176469.197418:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176469.197421:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176469.197425:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197429:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176469.197432:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197435:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176469.197439:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176469.197443:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176469.197447:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176469.197452:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176469.197454:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176469.197458:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.197462:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176469.197464:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176469.197468:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.197472:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176469.197479:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176469.197482:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176469.197486:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176469.197489:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176469.197491:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176469.197494:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176469.197498:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176469.197505:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176469.197513:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176469.197522:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235496832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176469.197540:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176469.197546:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176469.197555:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235496832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176469.197570:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176469.197577:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11153) 00000800:00000200:2.1:1786176469.197719:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176469.197721:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176469.197970:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176469.197972:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176469.198193:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176469.198195:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176469.198443:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176469.198445:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00010000:00000001:0.0:1786176470.092450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176470.092457:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176470.092462:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176470.092467:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176470.092473:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176470.092476:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176470.092480:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176470.092483:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176470.092486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176470.092487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176470.092490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176470.092492:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176470.092493:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176470.092495:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176470.092496:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176470.092497:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176470.092499:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176470.092500:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176471.116469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176471.116474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176471.116482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176471.116487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176471.116494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176471.116499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176472.140472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176472.140477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176472.140483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176472.140486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176472.140490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176472.140493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176473.100468:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305819904, at 4305819968 00000100:00000010:0.0:1786176473.100488:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c6c0. 00010000:00000001:0.0:1786176473.164473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176473.164478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176473.164484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176473.164488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176473.164493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176473.164497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176474.188457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176474.188462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176474.188466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176474.188468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176474.188472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176474.188476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176474.316477:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176474.316486:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176474.316488:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176474.316491:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176474.316496:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176474.316501:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11148 00000400:00000200:0.0:1786176474.316509:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176474.316515:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176474.316517:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176474.316519:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316522:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176474.316524:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316526:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176474.316529:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176474.316531:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176474.316533:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176474.316536:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176474.316537:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176474.316539:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176474.316541:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316543:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176474.316544:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316545:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176474.316548:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176474.316550:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316551:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176474.316554:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176474.316557:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176474.316560:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.316563:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176474.316565:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176474.316575:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176474.316579:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176474.316584:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176474.316586:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.316590:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176474.316593:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176474.316597:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176474.316606:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176474.316612:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176474.316620:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235496960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176474.316628:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176474.316631:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176474.316635:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235496960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176474.316653:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176474.316655:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176474.316658:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176474.316659:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176474.316660:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176474.316662:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176474.316664:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11148 00000400:00000200:0.0:1786176474.316667:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176474.316669:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176474.316671:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176474.316673:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316675:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176474.316676:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316677:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176474.316679:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176474.316680:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176474.316681:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176474.316682:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176474.316684:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176474.316685:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176474.316688:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316690:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176474.316692:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316693:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176474.316695:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176474.316697:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316698:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176474.316699:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176474.316704:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176474.316706:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.316708:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176474.316711:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176474.316712:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.316714:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176474.316716:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176474.316719:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176474.316720:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176474.316721:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.316722:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176474.316723:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176474.316725:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176474.316730:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786176474.316747:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176474.316763:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235497088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176474.316828:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176474.316831:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176474.316844:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235497088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176474.316877:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176474.316884:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176474.316890:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176474.316892:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176474.316899:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176474.316913:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176474.316926:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11148 00000400:00000200:0.0:1786176474.316931:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176474.316942:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176474.316945:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176474.316948:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316953:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176474.316958:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.316966:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176474.316969:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176474.316969:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176474.316972:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176474.316977:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176474.316981:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176474.316983:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176474.317002:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317013:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176474.317027:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317037:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176474.317046:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176474.317056:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317064:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176474.317067:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176474.317070:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176474.317081:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.317085:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176474.317087:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176474.317094:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.317097:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176474.317101:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176474.317108:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176474.317117:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176474.317120:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.317126:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176474.317130:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176474.317133:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176474.317143:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176474.317154:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176474.317167:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235497216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176474.317204:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176474.317206:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176474.317220:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235497216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176474.317240:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176474.317250:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.1:1786176474.317257:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:0.0:1786176474.317259:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:2.1:1786176474.317262:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000100:00000001:0.0:1786176474.317264:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176474.317270:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176474.317277:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176474.317284:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11148 00000400:00000200:0.0:1786176474.317299:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176474.317309:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176474.317313:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176474.317344:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317349:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176474.317400:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317405:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176474.317411:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176474.317415:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176474.317421:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176474.317427:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176474.317434:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176474.317448:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176474.317459:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317460:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176474.317466:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317474:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176474.317486:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176474.317495:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317498:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176474.317500:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176474.317503:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176474.317508:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.317510:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176474.317513:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176474.317515:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.317517:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176474.317529:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176474.317532:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176474.317533:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176474.317535:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.317537:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176474.317539:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176474.317543:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176474.317548:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176474.317553:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176474.317566:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235497344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176474.317633:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176474.317640:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176474.317648:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235497344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176474.317665:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176474.317669:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176474.317674:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176474.317676:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176474.317679:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176474.317684:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176474.317689:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11148 00000400:00000200:0.0:1786176474.317700:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176474.317705:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176474.317708:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176474.317716:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317729:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176474.317736:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317745:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176474.317749:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176474.317755:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176474.317767:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176474.317779:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176474.317791:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176474.317796:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176474.317810:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317820:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176474.317828:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317831:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176474.317836:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176474.317848:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176474.317858:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176474.317864:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176474.317872:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176474.317876:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.317879:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176474.317884:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176474.317895:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.317899:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176474.317905:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176474.317915:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176474.317929:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176474.317932:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176474.317938:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176474.317941:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176474.317947:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176474.317964:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176474.317973:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176474.317984:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235497472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176474.318009:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176474.318018:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176474.318035:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235497472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176474.318064:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176474.318070:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11158) 00000800:00000200:2.1:1786176474.318204:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176474.318206:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786176474.318447:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176474.318449:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786176474.318686:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176474.318690:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786176474.318891:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176474.318893:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00010000:00000001:0.0:1786176475.212463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176475.212467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176475.212472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176475.212474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176475.212478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176475.212479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176476.236456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176476.236461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176476.236468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176476.236470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176476.236475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176476.236477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176477.260500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176477.260508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176477.260516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176477.260518:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176477.260523:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176477.260524:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176477.900458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176477.900465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176477.900467:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176477.900471:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176477.900474:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176477.900477:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176477.900479:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176477.900481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176477.900483:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176478.220435:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305825025, at 4305825088 00000100:00000010:0.0:1786176478.220446:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c840. 00010000:00000001:0.0:1786176478.284442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176478.284448:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176478.284457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176478.284462:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176478.284467:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176478.284473:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176478.284477:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176478.284479:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176478.284480:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176478.284482:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176478.284486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176478.284487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176478.924428:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176478.924435:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176478.924437:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176478.924442:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176478.924446:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176478.924449:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176478.924451:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176478.924454:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176478.924457:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176479.308429:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176479.308432:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176479.308438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176479.308440:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176479.308443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176479.308445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176479.436471:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176479.436480:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176479.436482:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176479.436485:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176479.436488:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176479.436495:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11153 00000400:00000200:3.0:1786176479.436502:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176479.436506:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176479.436509:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176479.436511:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436514:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176479.436516:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436518:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176479.436521:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176479.436523:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176479.436525:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176479.436528:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176479.436530:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176479.436532:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176479.436534:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436535:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176479.436536:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436537:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176479.436539:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176479.436541:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436543:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176479.436545:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176479.436547:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176479.436549:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.436551:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176479.436552:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176479.436560:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176479.436562:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176479.436565:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176479.436566:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.436569:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176479.436570:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176479.436574:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176479.436580:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176479.436585:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176479.436592:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235497600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176479.436600:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176479.436604:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176479.436607:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235497600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176479.436628:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176479.436630:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176479.436633:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176479.436634:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176479.436635:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176479.436637:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176479.436638:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11153 00000400:00000200:3.0:1786176479.436640:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176479.436643:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176479.436644:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176479.436647:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436648:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176479.436650:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436652:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176479.436653:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176479.436655:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176479.436657:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176479.436659:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176479.436661:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176479.436664:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176479.436669:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436671:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176479.436674:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436678:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176479.436681:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176479.436683:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436684:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176479.436687:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176479.436693:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176479.436697:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.436702:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176479.436706:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176479.436710:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.436712:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176479.436716:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176479.436719:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176479.436722:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176479.436724:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.436727:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176479.436729:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176479.436733:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176479.436736:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176479.436741:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176479.436750:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235497728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176479.436756:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176479.436757:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176479.436763:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235497728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176479.436772:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176479.436775:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176479.436779:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176479.436781:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176479.436784:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176479.436785:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176479.436788:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11153 00000400:00000200:3.0:1786176479.436793:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176479.436795:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176479.436797:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176479.436801:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436803:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176479.436807:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436809:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176479.436812:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176479.436814:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176479.436817:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176479.436820:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176479.436823:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176479.436828:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176479.436833:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436838:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176479.436842:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436844:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176479.436850:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176479.436852:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176479.436855:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176479.436858:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176479.436860:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176479.436862:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.436866:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176479.436869:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176479.436873:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.436875:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176479.436881:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176479.436885:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176479.436889:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176479.436892:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.436896:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176479.436899:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176479.436902:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176479.436909:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176479.436917:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176479.436926:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235497856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176479.436935:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176479.436937:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176479.436942:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235497856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176479.436959:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176479.436962:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176479.436966:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176479.436967:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176479.436969:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176479.436973:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176479.436976:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11153 00000400:00000200:3.0:1786176479.436984:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176479.436989:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176479.436993:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176479.436997:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:2.1:1786176479.436997:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176479.437003:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1786176479.437004:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000040:3.0:1786176479.437006:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.437010:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176479.437014:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176479.437016:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176479.437019:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176479.437021:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176479.437025:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176479.437027:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176479.437031:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.437034:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176479.437039:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.437042:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176479.437044:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176479.437050:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176479.437053:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176479.437057:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176479.437060:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176479.437064:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.437065:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176479.437067:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176479.437070:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.437072:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176479.437076:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176479.437080:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176479.437084:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176479.437086:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.437089:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176479.437091:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176479.437095:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176479.437099:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176479.437103:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176479.437111:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235497984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176479.437119:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176479.437122:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176479.437128:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235497984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176479.437140:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176479.437142:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176479.437146:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176479.437147:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176479.437149:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176479.437152:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176479.437156:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11153 00000400:00000200:3.0:1786176479.437160:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176479.437165:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176479.437169:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176479.437174:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.437176:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176479.437181:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.437183:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176479.437186:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176479.437189:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176479.437191:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176479.437192:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176479.437194:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176479.437195:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176479.437197:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.437198:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176479.437200:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176479.437202:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176479.437204:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176479.437208:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176479.437210:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176479.437211:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176479.437212:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176479.437217:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.437220:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176479.437222:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176479.437225:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.437228:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176479.437232:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176479.437234:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176479.437238:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176479.437240:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176479.437242:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176479.437246:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176479.437247:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176479.437253:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176479.437257:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176479.437265:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235498112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176479.437300:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176479.437304:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176479.437312:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235498112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176479.437325:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176479.437330:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11164) 00000800:00000200:2.1:1786176479.437528:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176479.437530:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176479.437772:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176479.437774:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176479.437996:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176479.437998:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176479.438258:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176479.438261:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1786176479.948418:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176479.948425:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176479.948427:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176479.948432:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176479.948436:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176479.948438:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176479.948440:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176479.948442:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176479.948445:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176480.332439:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176480.332447:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176480.332453:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176480.332460:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176480.332465:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176480.332468:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176480.332470:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176480.332473:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176480.332478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176480.332480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176480.332483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176480.332485:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176480.332486:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176480.332488:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176480.332490:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176480.332492:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176480.332494:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176480.332495:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176480.972427:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176480.972435:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176480.972438:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176480.972444:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176480.972449:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176480.972452:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176480.972454:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176480.972458:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176480.972461:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176481.356446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176481.356450:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176481.356454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176481.356455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176481.356460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176481.356463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176481.996415:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176481.996422:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176481.996424:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176481.996429:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176481.996433:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176481.996437:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176481.996440:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176481.996443:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176481.996445:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176482.380439:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176482.380443:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176482.380449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176482.380452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176482.380456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176482.380459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176483.340406:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305830144, at 4305830208 00000100:00000010:0.0:1786176483.340420:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cea0. 00010000:00000001:0.0:1786176483.404449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176483.404455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176483.404463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176483.404467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176483.404474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176483.404478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176484.428474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176484.428479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176484.428485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176484.428486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176484.428490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176484.428491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176484.557098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176484.557105:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176484.557775:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176484.557777:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176484.558044:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176484.558047:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176484.558327:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176484.558328:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176484.558524:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176484.558526:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1786176485.068491:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176485.068497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176485.068499:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176485.068504:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176485.068507:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176485.068511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176485.068513:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176485.068516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176485.068518:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176485.452450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176485.452456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176485.452461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176485.452462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176485.452465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176485.452466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176486.092502:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176486.092516:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176486.092519:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176486.092526:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176486.092532:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176486.092538:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176486.092542:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176486.092546:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176486.092549:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176486.476435:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176486.476441:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176486.476445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176486.476446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176486.476448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176486.476449:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176487.117007:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176487.117016:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176487.117018:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176487.117024:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176487.117029:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176487.117033:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176487.117036:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176487.117039:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176487.117041:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176487.500438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176487.500443:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176487.500448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176487.500452:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176487.500456:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176487.500461:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176487.500466:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176487.500470:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176487.500473:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176487.500474:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176487.500477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176487.500478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176488.140639:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176488.140648:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176488.140651:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176488.140657:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176488.140661:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176488.140665:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176488.140668:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176488.140671:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176488.140674:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176488.460430:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305835264, at 4305835328 00000100:00000010:0.0:1786176488.460445:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c1e0. 00010000:00000001:0.0:1786176488.524471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176488.524493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176488.524500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176488.524504:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176488.524508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176488.524511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176489.164539:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176489.164551:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176489.164554:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176489.164562:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176489.164567:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176489.164572:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176489.164575:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176489.164579:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176489.164583:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176489.548440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176489.548448:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176489.548451:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176489.548455:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176489.548460:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176489.548462:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176489.548467:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176489.548470:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176489.548475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176489.548479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176489.548487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176489.548492:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176489.548494:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176489.548498:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176489.548502:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176489.548505:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176489.548519:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176489.548540:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176489.677077:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176489.677088:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176489.677824:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176489.677825:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176489.678009:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176489.678010:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176489.678317:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176489.678319:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176489.678542:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176489.678544:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1786176490.188495:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176490.188506:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176490.188513:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176490.188520:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176490.188525:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176490.188530:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176490.188534:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176490.188539:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176490.188543:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176490.572486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176490.572493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176490.572516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176490.572521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176490.572527:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176490.572529:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176491.596455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176491.596464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176491.596474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176491.596479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176491.596486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176491.596492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176492.620502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176492.620509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176492.620518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176492.620521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176492.620529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176492.620533:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176493.580434:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305840384, at 4305840448 00000100:00000010:0.0:1786176493.580451:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62a20. 00010000:00000001:0.0:1786176493.644447:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176493.644453:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176493.644460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176493.644463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176493.644470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176493.644472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176494.668494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176494.668501:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176494.668512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176494.668516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176494.668523:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176494.668527:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176494.796564:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176494.796578:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176494.796581:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176494.796586:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176494.796592:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176494.796602:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11169 00000400:00000200:3.0:1786176494.796615:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176494.796625:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176494.796632:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176494.796638:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.796645:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176494.796650:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.796654:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176494.796662:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176494.796666:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176494.796670:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176494.796676:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176494.796681:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176494.796687:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176494.796694:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.796699:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176494.796702:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.796706:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176494.796712:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176494.796718:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176494.796721:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176494.796725:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176494.796729:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176494.796733:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.796736:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176494.796739:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176494.796752:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176494.796757:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176494.796762:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176494.796764:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.796768:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176494.796772:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176494.796778:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176494.796787:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176494.796795:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176494.796807:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235499520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176494.796822:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176494.796827:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176494.796834:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235499520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176494.796864:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176494.796869:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176494.796874:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176494.796876:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176494.796878:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176494.796882:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176494.796885:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11169 00000400:00000200:3.0:1786176494.796891:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176494.796895:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176494.796898:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176494.796903:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.796906:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176494.796910:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.796913:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176494.796917:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176494.796919:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176494.796922:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176494.796925:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176494.796927:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176494.796930:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176494.796934:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.796937:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176494.796941:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.796950:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176494.796956:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176494.796964:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176494.796972:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176494.796975:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176494.796984:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176494.796994:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.797001:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176494.797009:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176494.797016:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.797024:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176494.797031:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176494.797038:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176494.797045:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176494.797051:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.797057:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176494.797062:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176494.797068:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176494.797078:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176494.797102:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176494.797111:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235499648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176494.797167:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176494.797175:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176494.797190:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235499648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176494.797220:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176494.797228:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176494.797245:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176494.797250:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176494.797257:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176494.797265:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176494.797280:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11169 00000400:00000200:3.0:1786176494.797295:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176494.797305:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176494.797315:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176494.797327:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.797337:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176494.797346:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.797352:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176494.797394:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176494.797395:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176494.797397:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176494.797400:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176494.797403:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176494.797406:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176494.797411:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.797414:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176494.797416:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.797420:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176494.797448:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176494.797462:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176494.797477:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176494.797490:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176494.797502:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176494.797513:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.797523:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176494.797530:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176494.797540:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.797546:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176494.797553:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176494.797557:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176494.797566:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176494.797571:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.797578:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176494.797581:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176494.797584:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176494.797593:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176494.797601:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176494.797618:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235499776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1786176494.797648:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176494.797651:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1786176494.797662:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1786176494.797663:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176494.797684:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235499776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176494.797715:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176494.797724:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176494.797730:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176494.797739:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176494.797742:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176494.797746:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176494.797757:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11169 00000400:00000200:3.0:1786176494.797768:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176494.797775:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176494.797780:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176494.797789:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.797794:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176494.797801:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.797808:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176494.797815:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176494.797820:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176494.797825:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176494.797830:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176494.797835:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176494.797843:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176494.797854:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.797863:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176494.797870:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.797877:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176494.797886:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176494.797894:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176494.797903:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176494.797908:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176494.797918:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176494.797922:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.797925:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176494.797929:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176494.797938:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.797941:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176494.797952:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176494.797960:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176494.797964:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176494.797972:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.797976:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176494.797979:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176494.797985:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176494.797995:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176494.798005:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176494.798016:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235499904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176494.798038:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176494.798045:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176494.798056:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235499904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176494.798077:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176494.798102:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176494.798112:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176494.798115:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176494.798147:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176494.798156:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176494.798167:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11169 00000400:00000200:3.0:1786176494.798186:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176494.798201:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176494.798210:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176494.798224:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.798233:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176494.798244:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.798253:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176494.798264:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176494.798271:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176494.798277:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176494.798284:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176494.798293:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176494.798301:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176494.798316:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.798327:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176494.798335:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176494.798341:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176494.798348:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176494.798381:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176494.798386:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176494.798388:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176494.798393:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176494.798401:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.798406:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176494.798411:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176494.798438:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.798445:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176494.798459:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786176494.798469:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176494.798482:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176494.798490:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176494.798501:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176494.798508:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176494.798517:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176494.798533:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176494.798543:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176494.798559:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235500032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176494.798580:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176494.798588:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176494.798600:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235500032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176494.798630:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176494.798647:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11179) 00000800:00000200:1.1:1786176494.798918:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176494.798924:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176494.799314:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176494.799317:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176494.799841:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176494.799844:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176494.800197:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176494.800200:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1786176495.692539:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176495.692551:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176495.692569:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176495.692577:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176495.692591:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176495.692599:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176496.716514:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176496.716523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176496.716544:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176496.716550:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176496.716560:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176496.716567:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176497.740449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176497.740456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176497.740462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176497.740465:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176497.740468:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176497.740472:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176497.740476:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176497.740478:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176497.740481:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176497.740484:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176497.740488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176497.740489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176498.700431:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305845505, at 4305845568 00000100:00000010:0.0:1786176498.700447:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62300. 00010000:00000001:0.0:1786176498.764449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176498.764453:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176498.764462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176498.764464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176498.764468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176498.764471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176499.788451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176499.788457:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176499.788461:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176499.788465:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176499.788470:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176499.788474:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176499.788480:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176499.788486:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176499.788493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176499.788495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176499.788498:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176499.788501:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176499.788502:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176499.788504:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176499.788506:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176499.788508:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176499.788510:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176499.788513:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176499.916472:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176499.916485:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176499.916488:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176499.916491:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176499.916496:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176499.916504:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11174 00000400:00000200:0.0:1786176499.916513:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176499.916519:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176499.916523:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176499.916526:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916530:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176499.916532:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916534:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176499.916537:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176499.916539:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176499.916541:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176499.916544:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176499.916545:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176499.916547:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176499.916549:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916550:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176499.916551:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916552:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176499.916555:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176499.916557:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916559:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176499.916561:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176499.916563:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176499.916565:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.916568:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176499.916570:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176499.916578:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176499.916581:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176499.916584:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176499.916586:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.916588:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176499.916590:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176499.916594:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176499.916603:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176499.916608:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176499.916616:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235500160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176499.916624:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176499.916628:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176499.916631:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235500160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176499.916653:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176499.916657:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176499.916660:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176499.916662:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176499.916664:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176499.916667:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176499.916670:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11174 00000400:00000200:0.0:1786176499.916674:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176499.916678:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176499.916680:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176499.916683:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916686:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176499.916688:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916690:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176499.916692:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176499.916693:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176499.916694:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176499.916696:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176499.916697:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176499.916699:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176499.916701:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916702:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176499.916704:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916705:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176499.916707:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176499.916710:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916712:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176499.916713:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176499.916718:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176499.916721:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.916724:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176499.916728:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176499.916732:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.916735:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176499.916740:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176499.916745:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176499.916750:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176499.916752:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.916756:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176499.916759:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176499.916762:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176499.916768:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176499.916774:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176499.916782:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235500288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176499.916797:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176499.916803:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176499.916811:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235500288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176499.916829:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176499.916834:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176499.916840:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176499.916845:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176499.916847:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176499.916851:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176499.916858:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11174 00000400:00000200:0.0:1786176499.916871:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176499.916881:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176499.916887:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176499.916891:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916892:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176499.916893:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916894:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176499.916899:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176499.916901:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176499.916905:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176499.916908:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176499.916912:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176499.916914:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176499.916920:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916924:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176499.916926:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916928:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176499.916933:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176499.916938:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176499.916941:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176499.916944:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176499.916949:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176499.916953:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.916955:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176499.916957:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176499.916978:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.916983:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176499.916992:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176499.917003:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176499.917012:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176499.917016:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.917022:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176499.917025:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176499.917028:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176499.917036:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176499.917042:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176499.917051:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235500416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176499.917065:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176499.917085:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176499.917089:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:0.0:1786176499.917095:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235500416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1786176499.917097:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000001:0.0:1786176499.917114:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176499.917118:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176499.917122:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176499.917124:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176499.917128:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176499.917136:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176499.917143:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11174 00000400:00000200:0.0:1786176499.917154:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176499.917162:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176499.917167:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176499.917174:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.917180:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176499.917186:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.917191:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176499.917193:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176499.917198:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176499.917201:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176499.917205:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176499.917210:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176499.917216:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176499.917225:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.917232:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176499.917236:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.917239:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176499.917242:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176499.917248:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176499.917252:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176499.917258:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176499.917261:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176499.917268:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.917270:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176499.917272:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176499.917275:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.917280:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176499.917285:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176499.917288:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176499.917295:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176499.917297:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.917299:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176499.917301:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176499.917304:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176499.917310:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176499.917315:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176499.917325:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235500544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176499.917336:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176499.917341:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176499.917346:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235500544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176499.917412:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176499.917416:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176499.917422:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176499.917425:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176499.917428:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176499.917433:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176499.917438:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11174 00000400:00000200:0.0:1786176499.917443:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176499.917448:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176499.917450:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176499.917456:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.917462:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176499.917465:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.917469:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176499.917471:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176499.917474:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176499.917476:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176499.917481:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176499.917486:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176499.917492:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176499.917502:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.917507:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176499.917513:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176499.917518:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176499.917524:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176499.917530:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176499.917534:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176499.917538:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176499.917542:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176499.917546:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.917549:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176499.917551:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176499.917554:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.917557:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176499.917562:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176499.917566:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176499.917571:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176499.917573:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176499.917579:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176499.917580:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176499.917583:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176499.917589:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176499.917596:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176499.917605:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235500672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176499.917619:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176499.917623:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176499.917629:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235500672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176499.917647:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176499.917656:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11184) 00000800:00000200:1.1:1786176499.917880:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176499.917882:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176499.918146:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176499.918149:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176499.918442:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176499.918444:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176499.918627:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176499.918629:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00010000:00000001:0.0:1786176500.812452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176500.812455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176500.812462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176500.812463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176500.812467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176500.812468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176501.836449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176501.836455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176501.836469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176501.836476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176501.836482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176501.836485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176502.860453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176502.860457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176502.860466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176502.860467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176502.860474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176502.860476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176503.820467:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305850624, at 4305850688 00000100:00000010:0.0:1786176503.820485:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c622a0. 00010000:00000001:0.0:1786176503.884448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176503.884452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176503.884458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176503.884460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176503.884463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176503.884467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176504.908466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176504.908472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176504.908480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176504.908484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176504.908492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176504.908494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176505.036469:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176505.036477:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176505.036479:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176505.036482:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176505.036485:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176505.036490:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11179 00000400:00000200:3.0:1786176505.036497:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176505.036502:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176505.036505:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176505.036507:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036510:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176505.036512:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036514:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176505.036517:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176505.036519:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176505.036521:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176505.036523:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176505.036525:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176505.036526:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176505.036528:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036529:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176505.036531:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036532:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176505.036534:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176505.036536:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036538:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176505.036540:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176505.036542:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176505.036544:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.036546:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176505.036548:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176505.036556:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176505.036558:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176505.036561:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176505.036562:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.036565:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176505.036567:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176505.036570:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176505.036577:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176505.036582:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176505.036589:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235500800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176505.036597:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176505.036601:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176505.036604:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235500800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176505.036621:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176505.036624:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176505.036627:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176505.036628:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176505.036629:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176505.036631:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176505.036632:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11179 00000400:00000200:3.0:1786176505.036634:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176505.036636:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176505.036638:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176505.036640:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036642:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176505.036644:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036645:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176505.036647:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176505.036648:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176505.036649:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176505.036650:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176505.036652:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176505.036653:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176505.036656:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036657:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176505.036658:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036659:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176505.036661:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176505.036663:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036669:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176505.036672:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176505.036676:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176505.036680:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.036682:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176505.036685:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176505.036691:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.036695:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176505.036700:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786176505.036704:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176505.036708:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176505.036711:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.036713:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176505.036715:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176505.036717:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176505.036721:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176505.036727:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176505.036736:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235500928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176505.036748:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176505.036751:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176505.036756:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235500928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176505.036765:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176505.036768:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176505.036773:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176505.036777:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176505.036778:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176505.036781:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176505.036786:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11179 00000400:00000200:3.0:1786176505.036790:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176505.036796:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176505.036797:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176505.036800:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036805:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176505.036806:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036807:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176505.036809:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176505.036810:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176505.036813:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176505.036817:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176505.036821:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176505.036823:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176505.036828:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036830:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176505.036833:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036835:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176505.036840:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176505.036845:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036851:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176505.036855:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176505.036860:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176505.036865:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.036868:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176505.036870:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176505.036875:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.036877:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176505.036883:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786176505.036885:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176505.036890:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176505.036892:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.036895:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176505.036896:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176505.036898:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176505.036902:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176505.036906:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176505.036911:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235501056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176505.036920:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176505.036922:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176505.036927:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235501056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176505.036937:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176505.036941:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176505.036945:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176505.036948:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176505.036950:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176505.036957:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176505.036963:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11179 00000400:00000200:3.0:1786176505.036967:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176505.036971:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176505.036975:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176505.036981:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036986:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176505.036990:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.036992:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176505.036996:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176505.036998:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176505.037001:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176505.037002:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176505.037004:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176505.037009:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176505.037012:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.037015:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176505.037032:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.037035:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176505.037038:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176505.037043:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:1.1:1786176505.037044:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176505.037045:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176505.037048:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176505.037052:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176505.037054:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.037055:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176505.037057:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176505.037059:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.037060:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176505.037064:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786176505.037068:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176505.037072:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176505.037074:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.037079:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176505.037083:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176505.037087:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176505.037095:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176505.037100:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176505.037110:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235501184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176505.037123:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176505.037128:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176505.037136:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235501184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176505.037149:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176505.037153:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176505.037158:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176505.037162:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176505.037166:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176505.037172:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176505.037177:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11179 00000400:00000200:3.0:1786176505.037185:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176505.037192:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176505.037198:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176505.037205:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.037210:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176505.037215:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.037220:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176505.037224:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176505.037226:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176505.037227:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176505.037230:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176505.037236:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176505.037242:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176505.037250:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.037254:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176505.037261:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176505.037267:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176505.037272:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176505.037278:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176505.037280:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176505.037284:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176505.037287:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176505.037291:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.037295:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176505.037298:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176505.037301:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.037304:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176505.037310:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176505.037316:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176505.037322:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176505.037326:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176505.037331:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176505.037335:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176505.037340:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176505.037349:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176505.037387:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176505.037398:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235501312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176505.037418:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176505.037424:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176505.037440:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235501312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176505.037462:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176505.037469:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11189) 00000800:00000200:1.1:1786176505.037478:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176505.037650:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176505.037808:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176505.037974:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786176505.484711:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:0.0:1786176505.932458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176505.932469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176505.932483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176505.932488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176505.932499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176505.932505:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176506.956433:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176506.956440:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176506.956446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176506.956447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176506.956450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176506.956451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176507.980443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176507.980448:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176507.980454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176507.980459:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176507.980463:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176507.980470:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176507.980475:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176507.980481:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176507.980487:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176507.980490:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176507.980496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176507.980498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1786176508.108672:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:0.0:1786176508.108707:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1786176508.108715:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176508.108718:0:2380:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:0.0:1786176508.108723:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176508.108726:0:2380:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1786176508.108730:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176508.108734:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:0.0:1786176508.108740:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176508.108745:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176508.108751:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ee58 00000400:00000010:0.0:1786176508.108754:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ee58. 00000100:00000001:0.0:1786176508.108759:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176508.108769:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26325880 x1872930235500800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176508.108802:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176508.108807:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235500800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176508.108815:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176508.108818:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176508.108828:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176508.108830:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:0.0:1786176508.108846:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176508.108850:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176508.108854:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:0.0:1786176508.108856:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:0.0:1786176508.108860:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176508.108866:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26324000 x1872930235500928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176508.108879:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176508.108886:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324000 x1872930235500928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176508.108894:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176508.108897:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176508.108914:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176508.108920:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:0.0:1786176508.108933:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176508.108943:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176508.108949:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eaa0 00000400:00000010:0.0:1786176508.108954:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eaa0. 00000100:00000001:0.0:1786176508.108960:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176508.108967:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26326a00 x1872930235501056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176508.108977:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176508.108983:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326a00 x1872930235501056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176508.109069:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176508.109073:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176508.109088:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176508.109093:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:0.0:1786176508.109098:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176508.109110:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176508.109117:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e908 00000400:00000010:0.0:1786176508.109121:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e908. 00000100:00000001:0.0:1786176508.109127:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176508.109136:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26324e00 x1872930235501184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176508.109163:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176508.109168:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324e00 x1872930235501184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176508.109191:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176508.109196:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176508.109208:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176508.109214:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:0.0:1786176508.109220:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176508.109229:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176508.109239:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e6e8 00000400:00000010:0.0:1786176508.109243:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e6e8. 00000100:00000001:0.0:1786176508.109247:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176508.109256:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26324a80 x1872930235501312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176508.109275:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176508.109283:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324a80 x1872930235501312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176508.109309:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176508.109313:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1786176508.940473:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305855744, at 4305855808 00000100:00000010:0.0:1786176508.940489:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c620c0. 00010000:00000001:0.0:1786176509.004465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176509.004472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176509.004482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176509.004486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176509.004493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176509.004497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176510.028478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176510.028485:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176510.028489:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176510.028493:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176510.028497:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176510.028501:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176510.028503:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176510.028507:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176510.028510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176510.028513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176510.028518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176510.028520:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176510.028523:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176510.028525:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176510.028528:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176510.028530:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176510.028534:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176510.028536:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176510.156486:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176510.156494:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176510.156496:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176510.156499:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176510.156502:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176510.156508:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11184 00000400:00000200:0.0:1786176510.156515:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176510.156520:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176510.156522:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176510.156525:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.156528:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176510.156530:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.156532:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176510.156535:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176510.156536:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176510.156538:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176510.156541:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176510.156542:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176510.156544:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176510.156546:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.156547:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176510.156548:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.156550:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176510.156552:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176510.156555:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176510.156556:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176510.156559:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176510.156561:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176510.156563:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.156565:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176510.156566:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176510.156571:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786176510.156573:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176510.156577:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176510.156578:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.156580:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176510.156582:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176510.156586:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176510.156591:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176510.156597:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176510.156604:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235501440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176510.156612:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176510.156616:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176510.156619:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235501440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176510.156642:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176510.156646:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176510.156650:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176510.156651:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176510.156653:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176510.156655:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176510.156658:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11184 00000400:00000200:0.0:1786176510.156662:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176510.156665:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176510.156667:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176510.156677:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.156679:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176510.156682:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.156685:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176510.156687:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176510.156689:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176510.156694:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176510.156698:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176510.156704:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176510.156709:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176510.156716:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.156721:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176510.156727:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.156733:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176510.156739:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176510.156746:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176510.156753:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176510.156758:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176510.156765:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176510.156771:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.156776:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176510.156783:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176510.156790:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.156795:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176510.156802:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786176510.156808:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176510.156814:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176510.156815:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.156820:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176510.156824:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176510.156826:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176510.156832:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786176510.156838:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176510.156857:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235501568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176510.156873:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176510.156878:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176510.156887:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235501568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176510.156902:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176510.156908:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176510.156913:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176510.156917:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176510.156919:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176510.156926:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176510.156932:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11184 00000400:00000200:0.0:1786176510.157012:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176510.157021:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176510.157025:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176510.157031:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157035:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176510.157041:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157045:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176510.157052:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176510.157055:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000800:00000200:1.1:1786176510.157057:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:0.0:1786176510.157059:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:1.1:1786176510.157063:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000020:00000040:0.0:1786176510.157064:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176510.157070:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176510.157074:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176510.157082:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157086:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176510.157091:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157098:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176510.157104:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176510.157109:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157115:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176510.157119:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176510.157123:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176510.157128:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.157134:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176510.157138:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176510.157144:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.157146:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176510.157151:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786176510.157156:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176510.157163:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176510.157166:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.157171:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176510.157175:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176510.157179:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176510.157185:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176510.157190:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176510.157200:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235501696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176510.157212:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176510.157217:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176510.157223:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235501696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176510.157234:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176510.157239:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176510.157244:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176510.157249:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176510.157252:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176510.157259:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176510.157265:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11184 00000400:00000200:0.0:1786176510.157273:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176510.157281:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176510.157286:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176510.157293:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157296:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176510.157301:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157306:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176510.157311:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176510.157313:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176510.157317:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176510.157322:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176510.157327:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176510.157333:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176510.157342:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157348:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176510.157352:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157355:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176510.157406:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176510.157413:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157421:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176510.157426:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176510.157430:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176510.157436:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.157441:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176510.157446:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176510.157451:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.157454:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176510.157461:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786176510.157465:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176510.157471:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176510.157474:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.157479:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176510.157482:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176510.157486:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176510.157493:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176510.157499:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176510.157510:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235501824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176510.157526:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176510.157530:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176510.157538:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235501824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176510.157556:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176510.157563:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176510.157572:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176510.157577:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176510.157581:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176510.157586:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176510.157593:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11184 00000400:00000200:0.0:1786176510.157601:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176510.157610:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176510.157616:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176510.157624:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157629:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176510.157634:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157639:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176510.157644:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176510.157648:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176510.157653:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176510.157658:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176510.157663:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176510.157669:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176510.157674:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157679:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176510.157685:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157691:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176510.157696:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176510.157703:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176510.157708:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176510.157710:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176510.157714:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176510.157721:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.157726:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176510.157731:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176510.157738:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.157743:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176510.157750:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786176510.157754:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176510.157760:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176510.157764:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176510.157768:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176510.157772:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176510.157776:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176510.157784:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176510.157791:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176510.157801:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235501952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176510.157817:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176510.157824:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176510.157832:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235501952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176510.157850:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176510.157859:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11194) 00000800:00000200:1.1:1786176510.157949:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176510.157952:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176510.158289:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176510.158293:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176510.158655:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176510.158664:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176510.158944:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176510.158945:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1786176511.052453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176511.052459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176511.052467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176511.052470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176511.052476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176511.052480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176512.076438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176512.076444:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176512.076450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176512.076454:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176512.076459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176512.076462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176513.100476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176513.100480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176513.100489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176513.100493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176513.100500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176513.100503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176514.060474:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305860864, at 4305860928 00000100:00000010:0.0:1786176514.060489:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c624e0. 00010000:00000001:0.0:1786176514.124469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176514.124475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176514.124486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176514.124490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176514.124496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176514.124500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176515.148446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176515.148450:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176515.148456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176515.148459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176515.148465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176515.148468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176515.276492:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176515.276504:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176515.276507:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176515.276510:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176515.276515:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176515.276524:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11189 00000400:00000200:3.0:1786176515.276534:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176515.276542:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176515.276545:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176515.276548:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276550:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176515.276552:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276554:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176515.276557:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176515.276559:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176515.276561:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176515.276564:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176515.276567:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176515.276570:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176515.276573:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276575:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176515.276577:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276578:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176515.276581:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176515.276583:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276584:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176515.276587:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176515.276590:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176515.276593:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.276595:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176515.276597:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176515.276603:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176515.276606:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176515.276610:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176515.276612:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.276614:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176515.276617:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176515.276620:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176515.276626:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176515.276631:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176515.276639:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235502080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176515.276648:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176515.276651:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176515.276655:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235502080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176515.276675:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176515.276678:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176515.276680:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176515.276681:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176515.276683:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176515.276685:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176515.276686:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11189 00000400:00000200:3.0:1786176515.276689:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176515.276691:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176515.276693:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176515.276695:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276697:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176515.276698:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276700:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176515.276701:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176515.276702:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176515.276704:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176515.276705:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176515.276707:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176515.276709:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176515.276712:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276713:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176515.276715:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276717:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176515.276718:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176515.276721:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276726:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176515.276729:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176515.276734:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176515.276738:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.276740:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176515.276746:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176515.276751:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.276754:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176515.276759:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176515.276763:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176515.276766:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176515.276767:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.276771:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176515.276773:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176515.276775:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176515.276781:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176515.276785:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176515.276792:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235502208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176515.276800:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176515.276805:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176515.276812:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235502208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176515.276820:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176515.276824:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176515.276827:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176515.276830:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176515.276832:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176515.276835:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176515.276847:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11189 00000400:00000200:3.0:1786176515.276854:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176515.276860:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176515.276863:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176515.276869:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276873:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176515.276876:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276880:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176515.276884:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176515.276887:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176515.276891:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176515.276893:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176515.276898:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176515.276902:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176515.276907:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276910:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176515.276914:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276918:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176515.276935:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176515.276938:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176515.276940:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176515.276941:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176515.276943:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176515.276945:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.276946:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176515.276948:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176515.276949:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.276950:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176515.276953:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176515.276955:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176515.276956:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176515.276957:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.276958:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176515.276959:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176515.276961:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176515.276963:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176515.276966:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176515.276969:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235502336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176515.276975:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176515.276976:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176515.276982:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235502336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176515.276992:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176515.276998:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176515.277006:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176515.277009:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176515.277012:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176515.277021:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176515.277028:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11189 00000400:00000200:3.0:1786176515.277039:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176515.277047:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176515.277051:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176515.277058:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.277061:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176515.277065:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.277066:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176515.277069:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176515.277071:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176515.277076:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176515.277079:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176515.277084:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176515.277087:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000200:1.1:1786176515.277088:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786176515.277093:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:1.1:1786176515.277093:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1786176515.277096:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176515.277101:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.277104:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176515.277108:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176515.277112:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176515.277115:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176515.277117:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176515.277121:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176515.277125:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.277127:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176515.277130:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176515.277133:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.277135:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176515.277141:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176515.277145:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176515.277150:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176515.277152:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.277155:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176515.277158:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176515.277161:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176515.277166:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176515.277171:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176515.277179:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235502464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176515.277186:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176515.277190:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176515.277195:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235502464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176515.277203:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176515.277206:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176515.277208:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176515.277210:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176515.277222:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176515.277225:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176515.277230:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11189 00000400:00000200:3.0:1786176515.277236:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176515.277241:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176515.277245:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176515.277251:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.277256:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176515.277260:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.277264:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176515.277268:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176515.277270:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176515.277275:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176515.277277:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176515.277281:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176515.277284:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176515.277289:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.277294:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176515.277298:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176515.277302:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176515.277307:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176515.277312:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176515.277317:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176515.277321:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176515.277325:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176515.277327:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.277331:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176515.277334:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176515.277340:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.277350:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176515.277387:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176515.277393:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176515.277398:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176515.277399:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176515.277403:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176515.277405:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176515.277408:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176515.277413:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176515.277417:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176515.277425:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235502592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176515.277435:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176515.277439:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176515.277446:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235502592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176515.277461:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176515.277466:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11199) 00000800:00000200:1.1:1786176515.277642:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176515.277644:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176515.277909:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176515.277912:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176515.278172:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176515.278174:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176515.278478:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176515.278480:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1786176516.172486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176516.172495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176516.172504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176516.172508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176516.172514:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176516.172517:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176516.172596:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176516.172600:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176516.172604:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176517.196450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176517.196455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176517.196462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176517.196467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176517.196474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176517.196478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176518.220463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176518.220466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176518.220471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176518.220475:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176518.220479:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176518.220483:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176518.220496:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176518.220499:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176518.220503:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176518.220507:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176518.220510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176518.220513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176519.180479:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305865984, at 4305866048 00000100:00000010:0.0:1786176519.180498:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ea0. 00010000:00000001:0.0:1786176519.244472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176519.244484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176519.244494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176519.244499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176519.244508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176519.244512:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176519.884470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176519.884477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176519.884479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176519.884484:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176519.884487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176519.884490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176519.884493:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176519.884495:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176519.884497:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176520.268483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176520.268490:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176520.268494:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176520.268500:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176520.268505:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176520.268509:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176520.268515:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176520.268519:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176520.268526:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176520.268529:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176520.268533:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176520.268535:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176520.268536:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176520.268538:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176520.268540:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176520.268542:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176520.268544:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176520.268547:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176520.397087:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176520.397095:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176520.397672:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176520.397676:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176520.398056:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176520.398058:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176520.398314:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176520.398316:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176520.398518:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176520.398519:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:0.0:1786176520.908482:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176520.908490:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176520.908493:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176520.908498:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176520.908503:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176520.908507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176520.908510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176520.908513:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176520.908516:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176521.292468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176521.292475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176521.292482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176521.292485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176521.292489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176521.292493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176521.932425:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176521.932433:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176521.932435:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176521.932441:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176521.932445:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176521.932449:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176521.932453:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176521.932456:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176521.932460:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176522.316480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176522.316485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176522.316493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176522.316496:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176522.316500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176522.316502:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176522.956437:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176522.956444:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176522.956446:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176522.956451:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176522.956455:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176522.956459:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176522.956461:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176522.956463:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176522.956465:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176523.340462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176523.340467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176523.340473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176523.340477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176523.340481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176523.340483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176523.980467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176523.980473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176523.980475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176523.980480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176523.980485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176523.980488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176523.980491:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176523.980493:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176523.980496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176524.300488:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305871104, at 4305871168 00000100:00000010:0.0:1786176524.300506:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c780. 00010000:00000001:0.0:1786176524.364495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176524.364500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176524.364508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176524.364511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176524.364516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176524.364520:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176525.004490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176525.004498:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176525.004500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176525.004506:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176525.004510:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176525.004514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176525.004518:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176525.004522:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176525.004527:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176525.388450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176525.388459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176525.388465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176525.388466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176525.388470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176525.388471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176525.517080:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176525.517088:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176525.517765:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176525.517768:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176525.518076:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176525.518079:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176525.518354:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176525.518374:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176525.518768:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176525.518771:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00000001:0.0:1786176526.028469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176526.028476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176526.028478:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176526.028483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176526.028487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176526.028490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176526.028492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176526.028495:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176526.028497:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176526.412482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176526.412486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176526.412492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176526.412495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176526.412509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176526.412513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176527.052453:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176527.052460:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176527.052463:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176527.052469:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176527.052474:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176527.052478:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176527.052480:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176527.052483:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176527.052487:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176527.436425:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176527.436431:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176527.436436:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176527.436439:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176527.436441:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176527.436444:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176527.436448:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176527.436452:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176527.436455:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176527.436460:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176527.436467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176527.436470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176527.436527:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176527.436531:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176527.436533:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176528.076472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176528.076482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176528.076484:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176528.076491:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176528.076496:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176528.076500:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176528.076504:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176528.076507:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176528.076511:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176528.460462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176528.460467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176528.460485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176528.460488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176528.460495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176528.460498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176529.100442:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176529.100449:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176529.100450:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176529.100455:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176529.100458:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176529.100461:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176529.100463:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176529.100465:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176529.100467:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176529.420474:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305876224, at 4305876288 00000100:00000010:0.0:1786176529.420491:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62600. 00010000:00000001:0.0:1786176529.484471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176529.484478:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176529.484484:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176529.484488:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176529.484492:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176529.484495:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176529.484497:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176529.484508:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176529.484519:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176529.484522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176529.484528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176529.484531:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176529.484533:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176529.484534:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176529.484535:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176529.484536:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176529.484538:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176529.484539:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176529.484591:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176529.484594:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176529.484597:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176530.124467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176530.124473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176530.124475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176530.124480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176530.124484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176530.124488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176530.124490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176530.124492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176530.124495:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176530.508462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176530.508467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176530.508473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176530.508474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176530.508479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176530.508482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176530.636462:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176530.636471:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176530.636473:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176530.636476:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176530.636488:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176530.636497:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11205 00000400:00000200:3.0:1786176530.636504:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176530.636509:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176530.636511:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176530.636515:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636517:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176530.636519:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636521:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176530.636524:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176530.636525:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176530.636528:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176530.636530:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176530.636532:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176530.636533:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176530.636535:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636537:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176530.636538:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636539:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176530.636542:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176530.636544:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636546:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176530.636548:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176530.636550:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176530.636553:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.636554:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176530.636556:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176530.636564:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176530.636566:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176530.636569:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176530.636570:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.636573:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176530.636575:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176530.636579:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176530.636585:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176530.636590:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176530.636597:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235504000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176530.636606:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176530.636615:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176530.636619:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235504000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176530.636635:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176530.636638:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176530.636640:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176530.636641:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176530.636642:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176530.636644:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176530.636646:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11205 00000400:00000200:3.0:1786176530.636649:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176530.636652:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176530.636655:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176530.636657:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636660:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176530.636662:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636665:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176530.636668:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176530.636669:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176530.636672:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176530.636675:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176530.636677:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176530.636680:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176530.636684:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636690:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176530.636693:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636698:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176530.636703:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176530.636709:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636715:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176530.636720:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176530.636727:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176530.636733:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.636739:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176530.636746:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176530.636753:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.636757:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176530.636763:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176530.636768:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176530.636788:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176530.636789:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.636791:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176530.636792:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176530.636795:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176530.636799:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176530.636803:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176530.636813:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235504128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176530.636823:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176530.636828:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176530.636837:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235504128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176530.636854:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176530.636859:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176530.636866:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176530.636868:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176530.636870:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176530.636876:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176530.636883:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11205 00000400:00000200:3.0:1786176530.636892:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176530.636902:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176530.636906:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176530.636910:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636914:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176530.636919:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636923:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176530.636928:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176530.636930:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176530.636931:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176530.636934:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176530.636938:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176530.636939:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176530.636944:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636947:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176530.636952:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636954:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176530.636956:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176530.636960:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176530.636963:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176530.636967:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176530.636970:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176530.636972:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.636975:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176530.636979:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176530.636982:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.636984:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176530.636988:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176530.636992:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176530.636995:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176530.636998:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.637002:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176530.637005:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176530.637008:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176530.637012:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176530.637018:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176530.637027:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235504256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176530.637039:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176530.637042:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176530.637049:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235504256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176530.637067:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176530.637071:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176530.637077:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176530.637078:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176530.637079:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176530.637081:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176530.637086:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11205 00000400:00000200:3.0:1786176530.637091:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176530.637096:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176530.637100:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176530.637102:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.637106:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176530.637108:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.637111:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176530.637113:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176530.637114:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176530.637117:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:1.1:1786176530.637118:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786176530.637120:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176530.637126:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000200:1.1:1786176530.637127:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000020:00000001:3.0:1786176530.637128:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176530.637135:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.637137:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176530.637141:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.637146:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176530.637148:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176530.637154:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176530.637156:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176530.637158:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176530.637160:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176530.637163:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.637165:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176530.637168:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176530.637170:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.637174:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176530.637178:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176530.637181:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176530.637185:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176530.637187:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.637189:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176530.637191:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176530.637193:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176530.637196:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176530.637198:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176530.637201:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235504384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176530.637207:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176530.637211:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176530.637218:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235504384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176530.637238:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176530.637243:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176530.637247:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176530.637249:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176530.637252:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176530.637255:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176530.637259:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11205 00000400:00000200:3.0:1786176530.637267:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176530.637272:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176530.637276:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176530.637281:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.637285:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176530.637287:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.637292:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176530.637296:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176530.637297:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176530.637298:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176530.637300:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176530.637303:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176530.637309:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176530.637313:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.637315:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176530.637319:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176530.637322:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176530.637325:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176530.637330:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176530.637331:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176530.637333:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176530.637335:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176530.637339:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.637341:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176530.637343:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176530.637345:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.637348:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176530.637352:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176530.637354:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176530.637356:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176530.637381:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176530.637382:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176530.637383:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176530.637385:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176530.637389:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176530.637392:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176530.637395:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235504512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176530.637406:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176530.637408:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176530.637416:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235504512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176530.637425:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176530.637432:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11215) 00000800:00000200:1.1:1786176530.637743:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176530.637746:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176530.638074:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176530.638077:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176530.638383:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176530.638386:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176530.638671:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176530.638674:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:0.0:1786176531.148463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176531.148470:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176531.148472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176531.148476:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176531.148480:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176531.148483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176531.148485:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176531.148488:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176531.148490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176531.532515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176531.532523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176531.532534:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176531.532539:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176531.532549:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176531.532554:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176531.532635:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176531.532640:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176531.532645:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176532.556477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176532.556483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176532.556492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176532.556494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176532.556499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176532.556502:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176533.580421:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176533.580430:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176533.580437:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176533.580440:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176533.580445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176533.580446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176533.580504:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176533.580508:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176533.580512:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176534.540451:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305881344, at 4305881408 00000100:00000010:0.0:1786176534.540472:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62000. 00010000:00000001:0.0:1786176534.604448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176534.604455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176534.604463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176534.604468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176534.604477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176534.604481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176535.628504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176535.628510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176535.628517:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176535.628522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176535.628526:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176535.628529:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176535.628606:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176535.628611:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176535.628615:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176535.756525:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176535.756533:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176535.756535:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176535.756539:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176535.756543:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176535.756552:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11210 00000400:00000200:0.0:1786176535.756562:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176535.756569:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176535.756573:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176535.756578:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756582:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176535.756585:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756588:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176535.756593:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176535.756595:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176535.756598:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176535.756601:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176535.756603:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176535.756605:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176535.756608:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756609:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176535.756611:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756613:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176535.756615:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176535.756618:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756620:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176535.756622:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176535.756624:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176535.756627:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.756628:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176535.756630:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176535.756637:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786176535.756639:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176535.756642:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176535.756644:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.756646:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176535.756648:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176535.756652:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176535.756658:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176535.756663:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176535.756670:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235504640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176535.756678:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176535.756681:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176535.756685:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235504640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176535.756702:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176535.756704:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176535.756707:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176535.756708:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176535.756709:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176535.756712:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176535.756713:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11210 00000400:00000200:0.0:1786176535.756717:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176535.756719:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176535.756721:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176535.756738:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756740:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176535.756743:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756745:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176535.756747:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176535.756748:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176535.756749:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176535.756751:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176535.756752:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176535.756753:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176535.756755:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756757:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176535.756760:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756762:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176535.756767:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176535.756771:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756774:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176535.756781:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176535.756786:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176535.756790:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.756791:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176535.756795:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176535.756798:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.756799:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176535.756802:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786176535.756806:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176535.756810:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176535.756814:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.756816:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176535.756819:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176535.756824:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176535.756833:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176535.756841:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176535.756849:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235504768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176535.756859:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176535.756866:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176535.756872:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235504768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176535.756893:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176535.756901:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176535.756905:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176535.756908:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176535.756910:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176535.756914:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176535.756918:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11210 00000400:00000200:0.0:1786176535.756927:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176535.756933:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176535.756935:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176535.756938:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756951:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176535.756956:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756960:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176535.756963:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176535.756965:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176535.756968:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176535.756972:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176535.756978:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176535.756980:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176535.756984:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.756992:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176535.757003:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757006:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176535.757008:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176535.757010:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757011:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176535.757014:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176535.757017:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176535.757024:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.757033:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176535.757038:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176535.757044:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.757047:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176535.757053:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786176535.757059:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176535.757063:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176535.757063:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.757064:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176535.757065:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176535.757066:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176535.757069:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176535.757074:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176535.757087:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235504896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176535.757100:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176535.757106:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176535.757112:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235504896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176535.757123:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176535.757126:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176535.757138:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176535.757139:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176535.757140:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176535.757142:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176535.757152:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11210 00000400:00000200:0.0:1786176535.757157:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176535.757160:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176535.757162:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176535.757165:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757170:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176535.757175:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757177:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176535.757180:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176535.757182:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176535.757183:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176535.757188:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176535.757197:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176535.757201:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176535.757205:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757212:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176535.757214:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757220:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176535.757228:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176535.757238:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757243:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176535.757245:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176535.757248:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176535.757251:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.757258:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176535.757261:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176535.757265:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.757272:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176535.757276:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786176535.757281:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176535.757286:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176535.757286:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.757288:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176535.757289:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176535.757290:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176535.757292:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176535.757299:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176535.757309:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235505024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176535.757324:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176535.757327:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176535.757334:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235505024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1786176535.757425:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176535.757430:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000001:0.0:1786176535.757435:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176535.757441:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176535.757448:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176535.757450:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176535.757453:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176535.757457:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176535.757461:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11210 00000400:00000200:0.0:1786176535.757470:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176535.757479:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176535.757489:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176535.757495:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757503:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176535.757508:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757514:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176535.757516:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176535.757518:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176535.757523:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176535.757525:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176535.757529:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176535.757531:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176535.757541:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757548:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176535.757559:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757566:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176535.757578:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176535.757590:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176535.757593:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176535.757598:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176535.757601:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176535.757605:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.757610:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176535.757613:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176535.757623:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.757627:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176535.757636:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786176535.757640:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176535.757646:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176535.757653:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176535.757655:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176535.757657:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176535.757659:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176535.757663:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176535.757668:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176535.757680:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235505152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176535.757695:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176535.757697:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176535.757701:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235505152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176535.757711:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176535.757716:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11220) 00000800:00000200:1.1:1786176535.758023:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176535.758026:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176535.758225:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176535.758227:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176535.758431:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176535.758432:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176535.758618:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176535.758620:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00010000:00000001:0.0:1786176536.652462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176536.652466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176536.652473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176536.652477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176536.652480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176536.652483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176537.292465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176537.292473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176537.292475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176537.292481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176537.292485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176537.292490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176537.292492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176537.292495:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176537.292498:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176537.676461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176537.676467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176537.676475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176537.676480:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176537.676484:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176537.676489:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176537.676494:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176537.676498:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176537.676503:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176537.676505:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176537.676508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176537.676509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176537.676604:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176537.676618:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176537.676620:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176538.316468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176538.316476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176538.316478:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176538.316484:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176538.316489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176538.316493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176538.316496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176538.316499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176538.316502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176538.700471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176538.700476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176538.700483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176538.700487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176538.700491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176538.700494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176539.660470:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305886465, at 4305886528 00000100:00000010:0.0:1786176539.660490:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62960. 00010000:00000001:0.0:1786176539.724476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176539.724483:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176539.724487:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176539.724491:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176539.724494:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176539.724498:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176539.724500:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176539.724503:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176539.724507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176539.724510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176539.724515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176539.724516:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176539.724519:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176539.724522:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176539.724524:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176539.724527:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176539.724529:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176539.724530:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176539.724583:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176539.724586:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176539.724589:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176540.748471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176540.748480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176540.748492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176540.748495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176540.748501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176540.748506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176540.876458:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176540.876467:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176540.876469:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176540.876472:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176540.876476:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176540.876482:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11215 00000400:00000200:0.0:1786176540.876490:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176540.876495:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176540.876497:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176540.876499:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876502:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176540.876504:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876506:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176540.876509:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176540.876511:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176540.876513:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176540.876515:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176540.876517:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176540.876519:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176540.876520:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876522:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176540.876523:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876525:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176540.876527:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176540.876529:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876530:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176540.876532:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176540.876535:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176540.876537:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.876539:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176540.876541:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176540.876549:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176540.876552:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176540.876555:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176540.876556:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.876559:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176540.876562:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176540.876565:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176540.876570:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176540.876575:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176540.876582:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235505280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176540.876590:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176540.876593:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176540.876596:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235505280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176540.876614:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176540.876617:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176540.876619:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176540.876620:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176540.876622:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176540.876624:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176540.876625:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11215 00000400:00000200:0.0:1786176540.876628:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176540.876630:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176540.876632:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176540.876635:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876637:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176540.876639:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876641:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176540.876643:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176540.876644:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176540.876646:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176540.876649:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176540.876652:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176540.876656:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176540.876661:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876663:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176540.876667:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876672:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176540.876689:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176540.876695:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876697:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176540.876700:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176540.876707:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176540.876711:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.876715:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176540.876720:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176540.876724:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.876729:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176540.876733:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176540.876737:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176540.876742:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176540.876745:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.876748:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176540.876749:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176540.876751:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176540.876759:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786176540.876766:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176540.876780:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235505408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176540.876820:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176540.876826:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176540.876836:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235505408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176540.876853:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176540.876860:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176540.876868:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176540.876873:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176540.876877:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176540.876883:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176540.876890:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11215 00000400:00000200:0.0:1786176540.876908:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176540.876916:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176540.876921:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176540.876927:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876934:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176540.876938:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876942:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:1.1:1786176540.876945:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:0.0:1786176540.876946:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176540.876951:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000800:00000200:1.1:1786176540.876951:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000020:00000001:0.0:1786176540.876954:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176540.876958:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176540.876964:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176540.876968:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176540.876974:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876979:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176540.876984:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.876989:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176540.876993:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176540.877001:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176540.877006:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176540.877010:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176540.877015:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176540.877020:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.877025:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176540.877029:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176540.877035:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.877038:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176540.877046:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176540.877052:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176540.877058:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176540.877062:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.877066:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176540.877069:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176540.877073:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176540.877078:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176540.877084:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176540.877091:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235505536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176540.877104:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176540.877108:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176540.877117:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235505536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176540.877130:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176540.877135:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176540.877146:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176540.877150:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176540.877154:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176540.877161:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176540.877178:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11215 00000400:00000200:0.0:1786176540.877192:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176540.877201:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176540.877206:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176540.877213:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.877218:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176540.877224:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.877229:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176540.877234:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176540.877237:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176540.877242:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176540.877251:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176540.877255:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176540.877260:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176540.877268:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.877273:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176540.877278:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.877283:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176540.877289:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176540.877297:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176540.877303:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176540.877308:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176540.877312:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176540.877319:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.877322:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176540.877327:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176540.877335:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.877339:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176540.877344:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176540.877350:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176540.877355:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176540.877381:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.877383:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176540.877385:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176540.877389:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176540.877397:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176540.877404:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176540.877414:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235505664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176540.877427:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176540.877431:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176540.877439:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235505664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176540.877459:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176540.877464:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176540.877471:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176540.877476:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176540.877480:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176540.877484:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176540.877490:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11215 00000400:00000200:0.0:1786176540.877499:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176540.877506:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176540.877510:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176540.877518:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.877523:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176540.877528:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.877532:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176540.877537:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176540.877541:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176540.877544:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176540.877548:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176540.877554:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176540.877560:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176540.877567:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.877573:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176540.877579:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176540.877585:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176540.877590:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176540.877596:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176540.877602:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176540.877607:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176540.877612:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176540.877619:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.877623:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176540.877627:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176540.877632:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.877638:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176540.877645:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176540.877651:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176540.877657:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176540.877661:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176540.877665:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176540.877668:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176540.877684:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176540.877688:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000010:0.0:1786176540.877692:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000800:00000200:1.1:1786176540.877692:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:0.0:1786176540.877699:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176540.877709:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235505792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176540.877728:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176540.877733:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176540.877741:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235505792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176540.877758:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176540.877767:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11225) 00000800:00000200:1.1:1786176540.877935:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176540.877937:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176540.878194:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176540.878195:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176540.878393:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176540.878394:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00010000:00000001:0.0:1786176541.772474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176541.772479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176541.772487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176541.772490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176541.772494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176541.772495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176541.772553:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176541.772556:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176541.772559:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176542.796470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176542.796477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176542.796484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176542.796487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176542.796491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176542.796493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176543.820466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176543.820472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176543.820480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176543.820483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176543.820489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176543.820494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176543.820564:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176543.820569:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176543.820573:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176544.780461:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305891584, at 4305891648 00000100:00000010:0.0:1786176544.780478:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62e40. 00010000:00000001:0.0:1786176544.844428:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176544.844434:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176544.844440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176544.844441:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176544.844445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176544.844446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1786176545.420677:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:0.0:1786176545.868512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176545.868521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176545.868528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176545.868531:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176545.868535:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176545.868537:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176545.868592:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176545.868596:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176545.868599:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176545.997003:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176545.997007:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176545.997706:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176545.997708:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176545.997910:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176545.997912:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176545.998216:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176545.998220:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176545.998681:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176545.998683:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1786176546.892438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176546.892445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176546.892451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176546.892454:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176546.892459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176546.892462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176547.916471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176547.916476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176547.916485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176547.916491:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176547.916495:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176547.916500:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176547.916506:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176547.916510:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176547.916514:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176547.916517:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176547.916523:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176547.916525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176547.916585:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176547.916590:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176547.916593:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176548.940514:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176548.940521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176548.940527:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176548.940530:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176548.940537:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176548.940539:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176549.900444:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305896704, at 4305896768 00000100:00000010:0.0:1786176549.900458:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62a80. 00010000:00000001:0.0:1786176549.964466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176549.964473:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176549.964477:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176549.964480:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176549.964485:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176549.964487:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176549.964492:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176549.964495:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176549.964501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176549.964503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176549.964508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176549.964512:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176549.964514:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176549.964517:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176549.964520:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176549.964522:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176549.964525:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176549.964527:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176549.964619:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176549.964623:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176549.964626:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176550.988462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176550.988467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176550.988472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176550.988482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176550.988488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176550.988491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176551.116498:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176551.116506:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176551.116509:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176551.116512:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176551.116515:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176551.116522:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11225 00000400:00000200:3.0:1786176551.116529:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176551.116534:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176551.116536:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176551.116539:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116541:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176551.116543:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116545:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176551.116548:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176551.116550:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176551.116552:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176551.116554:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176551.116556:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176551.116558:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176551.116560:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116561:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176551.116562:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116564:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176551.116566:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176551.116568:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116570:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176551.116572:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176551.116588:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176551.116592:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.116594:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176551.116596:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176551.116607:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786176551.116610:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176551.116614:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176551.116615:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.116618:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176551.116620:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176551.116623:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176551.116630:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176551.116635:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176551.116643:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235506560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176551.116651:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176551.116655:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176551.116659:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235506560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176551.116676:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176551.116678:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176551.116681:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176551.116682:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176551.116683:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176551.116686:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176551.116687:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11225 00000400:00000200:3.0:1786176551.116690:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176551.116692:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176551.116694:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176551.116696:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116698:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176551.116700:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116701:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176551.116704:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176551.116705:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176551.116705:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176551.116707:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176551.116709:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176551.116711:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176551.116713:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116716:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176551.116720:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116722:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176551.116727:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176551.116732:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116734:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176551.116736:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176551.116740:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176551.116745:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.116749:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176551.116753:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176551.116757:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.116760:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176551.116763:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786176551.116766:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176551.116768:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176551.116772:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.116773:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176551.116774:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176551.116778:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176551.116783:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176551.116787:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176551.116795:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235506688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176551.116805:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176551.116809:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176551.116815:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235506688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176551.116830:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176551.116833:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176551.116838:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176551.116842:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176551.116843:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176551.116844:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176551.116849:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11225 00000400:00000200:3.0:1786176551.116855:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176551.116857:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176551.116862:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176551.116865:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116866:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176551.116868:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116874:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176551.116875:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176551.116876:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176551.116877:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176551.116878:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176551.116880:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176551.116884:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176551.116887:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116892:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176551.116895:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116899:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176551.116901:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176551.116905:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176551.116907:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176551.116911:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176551.116913:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176551.116917:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.116921:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176551.116923:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176551.116928:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.116931:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176551.116936:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786176551.116939:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176551.116942:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176551.116944:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.116945:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176551.116947:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176551.116951:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176551.116956:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176551.116959:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176551.116967:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235506816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176551.116973:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176551.116977:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176551.116983:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235506816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176551.116995:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176551.116998:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176551.117002:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176551.117004:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176551.117006:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176551.117009:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176551.117016:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11225 00000400:00000200:3.0:1786176551.117027:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176551.117036:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176551.117042:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176551.117051:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.117057:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176551.117063:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.117067:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176551.117072:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176551.117073:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176551.117077:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176551.117081:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176551.117086:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176551.117092:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000200:1.1:1786176551.117098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786176551.117099:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.117103:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1786176551.117106:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000040:3.0:1786176551.117107:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.117112:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176551.117118:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176551.117125:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176551.117130:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176551.117132:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176551.117135:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176551.117140:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.117142:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176551.117146:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176551.117151:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.117154:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176551.117158:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176551.117163:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176551.117166:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176551.117169:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.117170:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176551.117172:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176551.117176:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176551.117180:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176551.117185:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176551.117193:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235506944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176551.117203:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176551.117205:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176551.117215:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235506944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176551.117242:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176551.117246:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176551.117251:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176551.117252:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176551.117254:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176551.117259:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176551.117263:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11225 00000400:00000200:3.0:1786176551.117270:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176551.117274:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176551.117276:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176551.117280:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.117282:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176551.117285:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.117288:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176551.117289:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176551.117290:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176551.117291:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176551.117294:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176551.117298:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176551.117302:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176551.117306:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.117309:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176551.117314:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176551.117319:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176551.117322:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176551.117328:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176551.117332:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176551.117337:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176551.117340:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176551.117343:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.117347:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176551.117351:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176551.117354:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.117381:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176551.117384:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176551.117386:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176551.117389:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176551.117392:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176551.117395:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176551.117398:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176551.117401:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176551.117405:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176551.117410:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176551.117418:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235507072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176551.117435:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176551.117438:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176551.117444:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235507072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176551.117457:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176551.117463:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11235) 00000800:00000200:1.1:1786176551.117651:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176551.117653:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176551.117838:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176551.117840:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176551.118019:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176551.118020:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176551.118261:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176551.118263:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1786176552.012475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176552.012482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176552.012488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176552.012491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176552.012496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176552.012499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176552.012613:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176552.012617:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176552.012620:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176553.036474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176553.036478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176553.036484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176553.036486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176553.036491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176553.036494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176554.060446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176554.060451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176554.060457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176554.060460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176554.060468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176554.060469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176554.060526:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176554.060529:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176554.060532:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176555.020479:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305901824, at 4305901888 00000100:00000010:0.0:1786176555.020496:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62660. 00010000:00000001:0.0:1786176555.084470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176555.084475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176555.084480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176555.084483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176555.084489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176555.084491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176556.108442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176556.108448:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176556.108457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176556.108461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176556.108470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176556.108472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176556.108552:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176556.108556:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176556.108559:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176556.236463:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176556.236472:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176556.236474:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176556.236477:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176556.236480:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176556.236487:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11230 00000400:00000200:3.0:1786176556.236494:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176556.236499:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176556.236501:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176556.236504:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236506:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176556.236508:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236510:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176556.236513:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176556.236515:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176556.236517:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176556.236520:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176556.236521:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176556.236523:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176556.236545:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236547:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176556.236548:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236550:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176556.236552:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176556.236554:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236557:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176556.236559:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176556.236562:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176556.236564:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.236566:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176556.236568:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176556.236576:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176556.236578:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176556.236582:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176556.236583:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.236591:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176556.236593:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176556.236597:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176556.236604:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176556.236610:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176556.236617:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235507200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176556.236626:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176556.236629:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176556.236633:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235507200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176556.236650:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176556.236652:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176556.236655:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176556.236656:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176556.236657:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176556.236659:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176556.236660:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11230 00000400:00000200:3.0:1786176556.236663:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176556.236665:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176556.236670:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176556.236673:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236675:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176556.236678:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236679:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176556.236681:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176556.236682:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176556.236683:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176556.236684:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176556.236686:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176556.236688:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176556.236694:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236698:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176556.236702:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236704:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176556.236708:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176556.236712:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236717:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176556.236720:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176556.236726:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176556.236731:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.236734:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176556.236738:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176556.236743:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.236747:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176556.236754:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176556.236758:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176556.236762:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176556.236764:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.236767:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176556.236768:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176556.236772:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176556.236776:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176556.236779:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176556.236787:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235507328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176556.236797:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176556.236801:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176556.236806:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235507328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176556.236815:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176556.236818:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176556.236823:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176556.236824:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176556.236826:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176556.236828:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176556.236832:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11230 00000400:00000200:3.0:1786176556.236837:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176556.236841:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176556.236843:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176556.236848:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236852:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176556.236855:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236859:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176556.236863:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176556.236866:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176556.236869:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176556.236873:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176556.236877:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176556.236879:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176556.236884:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236886:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176556.236889:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236894:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176556.236898:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176556.236901:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176556.236903:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176556.236907:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176556.236912:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176556.236914:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.236917:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176556.236919:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176556.236923:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.236924:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176556.236928:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176556.236932:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176556.236935:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176556.236938:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.236942:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176556.236945:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176556.236948:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176556.236952:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176556.236957:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176556.236965:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235507456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176556.236979:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176556.236982:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176556.236988:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235507456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176556.236998:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176556.237000:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176556.237005:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176556.237007:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176556.237011:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176556.237014:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176556.237020:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11230 00000800:00000200:1.1:1786176556.237024:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:3.0:1786176556.237028:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:1.1:1786176556.237030:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000400:00000200:3.0:1786176556.237033:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176556.237037:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176556.237042:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.237046:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176556.237049:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.237051:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176556.237057:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176556.237058:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176556.237060:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176556.237063:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176556.237065:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176556.237068:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176556.237073:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.237075:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176556.237077:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.237082:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176556.237085:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176556.237090:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176556.237094:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176556.237098:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176556.237102:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176556.237105:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.237108:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176556.237111:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176556.237116:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.237126:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176556.237132:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176556.237137:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176556.237143:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176556.237146:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.237150:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176556.237154:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176556.237157:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176556.237164:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176556.237170:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176556.237177:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235507584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176556.237190:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176556.237194:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176556.237200:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235507584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176556.237212:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176556.237215:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176556.237222:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176556.237225:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176556.237227:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176556.237231:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176556.237235:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11230 00000400:00000200:3.0:1786176556.237243:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176556.237248:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176556.237252:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176556.237258:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.237261:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176556.237266:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.237270:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176556.237273:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176556.237274:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176556.237276:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176556.237277:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176556.237278:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176556.237279:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176556.237282:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.237286:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176556.237290:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176556.237295:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176556.237300:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176556.237304:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176556.237308:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176556.237312:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176556.237315:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176556.237320:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.237324:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176556.237326:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176556.237329:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.237334:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176556.237339:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176556.237344:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176556.237349:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176556.237351:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176556.237354:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176556.237380:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176556.237383:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176556.237389:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176556.237393:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176556.237400:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235507712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176556.237410:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176556.237414:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176556.237420:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235507712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176556.237433:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176556.237439:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11240) 00000800:00000200:1.1:1786176556.237605:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176556.237607:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176556.237821:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176556.237823:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176556.238011:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176556.238013:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176556.238211:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176556.238213:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1786176557.132508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176557.132514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176557.132535:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176557.132537:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176557.132549:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176557.132553:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176557.773132:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176557.773141:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176557.773144:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176557.773149:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176557.773154:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176557.773159:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176557.773162:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176557.773166:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176557.773169:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176558.156470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176558.156476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176558.156481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176558.156483:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176558.156485:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176558.156488:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176558.156491:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176558.156494:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176558.156497:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176558.156501:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176558.156504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176558.156520:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176558.156588:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176558.156593:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176558.156597:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176559.180485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176559.180489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176559.180508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176559.180511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176559.180518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176559.180521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176560.140476:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305906944, at 4305907008 00000100:00000010:0.0:1786176560.140508:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62b40. 00010000:00000001:0.0:1786176560.204472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176560.204479:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176560.204484:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176560.204503:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176560.204508:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176560.204511:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176560.204516:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176560.204522:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176560.204531:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176560.204535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176560.204539:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176560.204541:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176560.204543:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176560.204545:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176560.204547:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176560.204549:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176560.204551:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176560.204554:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176560.204603:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176560.204605:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176560.204608:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176561.228488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176561.228494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176561.228503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176561.228506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176561.228522:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176561.228525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176561.357033:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176561.357041:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176561.357714:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176561.357716:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176561.357913:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176561.357915:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176561.358112:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176561.358114:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176561.358383:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176561.358385:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1786176562.252508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176562.252514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176562.252520:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176562.252523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176562.252529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176562.252532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176562.252597:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176562.252601:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176562.252604:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176563.276509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176563.276514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176563.276521:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176563.276525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176563.276531:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176563.276535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176564.300458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176564.300462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176564.300467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176564.300471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176564.300474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176564.300477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176564.300544:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176564.300548:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176564.300552:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176565.260435:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305912064, at 4305912128 00000100:00000010:0.0:1786176565.260468:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c120. 00010000:00000001:0.0:1786176565.324481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176565.324486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176565.324494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176565.324497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176565.324502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176565.324504:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176566.348448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176566.348453:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176566.348461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176566.348464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176566.348470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176566.348473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176566.348537:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176566.348540:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176566.348543:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176566.476495:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176566.476511:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176566.476513:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176566.476516:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176566.476520:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176566.476526:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11240 00000400:00000200:0.0:1786176566.476533:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176566.476538:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176566.476541:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176566.476543:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476546:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176566.476548:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476550:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176566.476553:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176566.476555:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176566.476557:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176566.476559:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176566.476561:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176566.476563:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176566.476565:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476566:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176566.476567:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476568:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176566.476570:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176566.476573:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476574:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176566.476576:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176566.476579:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176566.476581:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.476582:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176566.476584:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176566.476592:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176566.476594:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176566.476598:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176566.476599:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.476601:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176566.476604:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176566.476607:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176566.476616:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176566.476621:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176566.476629:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235508480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176566.476638:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176566.476641:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176566.476645:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235508480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176566.476663:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176566.476665:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176566.476668:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176566.476669:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176566.476670:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176566.476672:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176566.476674:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11240 00000400:00000200:0.0:1786176566.476676:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176566.476678:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176566.476680:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176566.476682:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476684:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176566.476687:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476689:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176566.476690:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176566.476691:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176566.476692:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176566.476694:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176566.476695:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176566.476696:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176566.476698:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476699:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176566.476701:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476703:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176566.476708:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176566.476711:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476714:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176566.476717:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176566.476722:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176566.476727:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.476730:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176566.476735:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176566.476738:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.476748:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176566.476752:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176566.476757:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176566.476762:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176566.476764:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.476766:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176566.476770:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176566.476773:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176566.476780:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176566.476786:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176566.476794:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235508608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176566.476805:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176566.476808:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176566.476815:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235508608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176566.476825:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176566.476830:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176566.476836:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176566.476839:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176566.476841:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176566.476847:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176566.476853:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11240 00000400:00000200:0.0:1786176566.476861:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176566.476867:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176566.476871:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176566.476884:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476886:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176566.476891:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476894:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176566.476895:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176566.476897:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176566.476900:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176566.476903:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176566.476906:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176566.476909:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176566.476912:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476913:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176566.476914:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476916:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176566.476917:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176566.476922:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176566.476926:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176566.476929:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176566.476934:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176566.476940:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.476944:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176566.476948:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176566.476952:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.476954:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176566.476960:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176566.476962:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176566.476965:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176566.476967:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.476970:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176566.476973:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176566.476976:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176566.476979:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176566.476985:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176566.476992:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235508736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176566.477001:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176566.477004:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176566.477008:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235508736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176566.477020:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176566.477023:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176566.477029:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176566.477032:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.1:1786176566.477032:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:0.0:1786176566.477034:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1786176566.477038:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000100:00000001:0.0:1786176566.477039:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176566.477045:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11240 00000400:00000200:0.0:1786176566.477051:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176566.477056:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176566.477058:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176566.477063:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.477066:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176566.477071:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.477074:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176566.477078:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176566.477080:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176566.477082:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176566.477084:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176566.477089:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176566.477091:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176566.477095:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.477096:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176566.477098:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.477099:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176566.477101:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176566.477105:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176566.477108:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176566.477109:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176566.477112:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176566.477115:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.477119:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176566.477123:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176566.477128:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.477132:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176566.477137:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176566.477140:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176566.477145:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176566.477147:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.477152:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176566.477154:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176566.477156:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176566.477162:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176566.477167:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176566.477173:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235508864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176566.477178:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176566.477179:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176566.477184:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235508864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176566.477198:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176566.477202:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176566.477207:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176566.477208:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176566.477210:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176566.477212:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176566.477217:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11240 00000400:00000200:0.0:1786176566.477222:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176566.477229:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176566.477233:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176566.477237:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.477240:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176566.477251:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.477255:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176566.477260:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176566.477262:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176566.477266:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176566.477269:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176566.477271:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176566.477275:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176566.477282:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.477286:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176566.477290:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176566.477293:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176566.477297:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176566.477301:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176566.477308:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176566.477312:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176566.477316:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176566.477319:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.477324:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176566.477327:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176566.477333:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.477339:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176566.477347:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176566.477353:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176566.477385:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176566.477388:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176566.477390:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176566.477393:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176566.477397:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176566.477406:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176566.477414:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176566.477444:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235508992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176566.477472:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176566.477477:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176566.477485:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235508992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176566.477499:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176566.477505:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11251) 00000800:00000200:2.1:1786176566.477659:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176566.477661:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176566.477848:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176566.477850:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176566.478057:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176566.478059:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176566.478270:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176566.478271:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00010000:00000001:0.0:1786176567.372478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176567.372483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176567.372491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176567.372494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176567.372497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176567.372499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176568.396478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176568.396482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176568.396487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176568.396490:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176568.396499:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176568.396502:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176568.396505:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176568.396507:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176568.396509:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176568.396512:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176568.396515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176568.396517:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176568.396576:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176568.396579:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176568.396582:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176569.420451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176569.420458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176569.420465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176569.420468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176569.420473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176569.420476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176570.380497:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305917184, at 4305917248 00000100:00000010:0.0:1786176570.380514:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c360. 00010000:00000001:0.0:1786176570.444467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176570.444483:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176570.444488:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176570.444495:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176570.444499:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176570.444501:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176570.444503:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176570.444505:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176570.444509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176570.444510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176570.444513:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176570.444514:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176570.444515:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176570.444516:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176570.444517:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176570.444518:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176570.444520:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176570.444522:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176570.444560:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176570.444563:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176570.444566:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176571.468463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176571.468468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176571.468473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176571.468484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176571.468492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176571.468494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176571.596471:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176571.596480:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176571.596482:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176571.596485:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176571.596489:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176571.596495:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11246 00000400:00000200:3.0:1786176571.596502:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176571.596507:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176571.596510:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176571.596512:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596515:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176571.596517:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596518:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176571.596521:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176571.596523:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176571.596525:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176571.596528:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176571.596529:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176571.596531:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176571.596533:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596534:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176571.596536:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596537:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176571.596539:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176571.596542:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596544:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176571.596556:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176571.596560:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176571.596563:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.596565:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176571.596568:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176571.596578:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176571.596582:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176571.596586:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176571.596588:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.596592:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176571.596594:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176571.596599:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176571.596608:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176571.596614:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176571.596626:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235509120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176571.596641:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176571.596644:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176571.596649:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235509120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176571.596671:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176571.596675:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176571.596679:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176571.596681:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176571.596683:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176571.596686:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176571.596689:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11246 00000400:00000200:3.0:1786176571.596695:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176571.596699:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176571.596701:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176571.596706:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596711:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176571.596724:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596730:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176571.596736:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176571.596739:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176571.596742:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176571.596746:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176571.596750:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176571.596755:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176571.596762:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596765:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176571.596771:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596778:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176571.596782:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176571.596786:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596789:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176571.596793:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176571.596806:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176571.596811:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.596815:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176571.596820:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176571.596825:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.596830:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176571.596840:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176571.596847:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176571.596854:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176571.596857:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.596861:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176571.596864:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176571.596867:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176571.596874:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176571.596879:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176571.596886:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235509248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176571.596899:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176571.596903:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176571.596910:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235509248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176571.596928:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176571.596933:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176571.596939:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176571.596942:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176571.596944:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176571.596951:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176571.596956:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11246 00000400:00000200:3.0:1786176571.596966:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176571.596973:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176571.596975:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176571.596980:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596981:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176571.596987:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.596990:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176571.596994:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:2.1:1786176571.596995:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786176571.596996:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176571.597001:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176571.597003:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176571.597005:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176571.597006:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176571.597009:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597011:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176571.597012:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597014:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176571.597018:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176571.597024:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597028:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176571.597031:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176571.597034:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176571.597037:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.597039:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176571.597041:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176571.597046:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.597048:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176571.597052:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176571.597056:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176571.597058:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176571.597062:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.597066:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176571.597067:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176571.597070:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176571.597077:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176571.597081:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176571.597086:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235509376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176571.597097:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176571.597100:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176571.597106:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235509376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176571.597123:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176571.597127:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176571.597134:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176571.597137:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176571.597140:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176571.597146:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176571.597152:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11246 00000400:00000200:3.0:1786176571.597159:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176571.597166:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176571.597169:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176571.597176:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597179:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176571.597184:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597187:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176571.597191:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176571.597193:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176571.597196:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176571.597198:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176571.597202:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176571.597206:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176571.597210:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597214:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176571.597219:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597222:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176571.597226:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176571.597231:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597235:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176571.597239:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176571.597241:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176571.597245:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.597249:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176571.597252:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176571.597254:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.597259:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176571.597262:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176571.597267:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176571.597273:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176571.597276:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.597281:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176571.597285:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176571.597289:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176571.597296:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176571.597304:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176571.597315:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235509504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176571.597337:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176571.597343:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176571.597352:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235509504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176571.597410:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176571.597416:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176571.597422:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176571.597426:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176571.597430:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176571.597437:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:2.1:1786176571.597442:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786176571.597444:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11246 00000400:00000200:3.0:1786176571.597453:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176571.597460:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176571.597466:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176571.597474:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597479:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176571.597485:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597493:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176571.597498:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176571.597501:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176571.597506:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176571.597511:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176571.597517:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176571.597522:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176571.597527:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597533:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176571.597542:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597547:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176571.597553:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176571.597562:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176571.597570:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176571.597575:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176571.597582:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176571.597587:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.597592:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176571.597598:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176571.597604:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.597610:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176571.597617:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176571.597624:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176571.597628:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176571.597631:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176571.597637:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176571.597640:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176571.597644:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176571.597652:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176571.597658:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176571.597668:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235509632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176571.597680:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176571.597685:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176571.597696:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235509632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176571.597715:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176571.597723:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11256) 00000800:00000200:2.1:1786176571.597780:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176571.597947:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176571.598131:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00010000:00000001:0.0:1786176572.492445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176572.492450:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176572.492456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176572.492457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176572.492460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176572.492461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176572.492504:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176572.492507:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176572.492510:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176573.516445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176573.516449:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176573.516453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176573.516456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176573.516461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176573.516464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176574.540467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176574.540474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176574.540483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176574.540488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176574.540497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176574.540501:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176574.540572:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176574.540577:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176574.540580:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176575.500467:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305922304, at 4305922368 00000100:00000010:0.0:1786176575.500483:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c420. 00010000:00000001:0.0:1786176575.564441:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176575.564448:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176575.564456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176575.564462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176575.564470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176575.564472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176576.588442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176576.588447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176576.588453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176576.588456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176576.588469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176576.588473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176576.588528:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176576.588531:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176576.588534:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176576.717169:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176576.717179:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176576.719202:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176576.719206:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176576.719645:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176576.719648:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176576.719893:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176576.719895:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176576.720072:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176576.720073:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1786176577.612474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176577.612480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176577.612486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176577.612491:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176577.612494:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176577.612498:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176577.612502:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176577.612504:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176577.612506:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176577.612511:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176577.612516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176577.612518:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176578.636473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176578.636487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176578.636492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176578.636495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176578.636501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176578.636504:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176578.636561:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176578.636564:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176578.636567:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176579.660483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176579.660489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176579.660497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176579.660499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176579.660504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176579.660506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176580.620459:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305927425, at 4305927488 00000100:00000010:0.0:1786176580.620473:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cc60. 00010000:00000001:0.0:1786176580.684513:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176580.684521:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176580.684526:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176580.684531:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176580.684537:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176580.684540:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176580.684543:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176580.684552:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176580.684560:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176580.684572:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176580.684584:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176580.684589:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176580.684590:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176580.684592:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176580.684594:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176580.684596:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176580.684597:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176580.684599:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176580.684665:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176580.684670:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176580.684675:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176581.708469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176581.708476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176581.708481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176581.708485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176581.708490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176581.708493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176581.837194:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176581.837203:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176581.837817:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176581.837820:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176581.838054:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176581.838057:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176581.838246:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176581.838247:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176581.838477:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176581.838479:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1786176582.732499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176582.732506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176582.732514:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176582.732520:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176582.732529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176582.732532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176582.732594:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176582.732598:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176582.732601:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.0:1786176583.308423:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:3.0:1786176583.308430:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:3.0:1786176583.308434:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1786176583.308597:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00010000:00000001:0.0:1786176583.756469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176583.756477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176583.756483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176583.756484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176583.756487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176583.756488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176584.780510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176584.780521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176584.780529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176584.780533:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176584.780538:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176584.780542:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176584.780604:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176584.780608:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176584.780611:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176585.740472:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305932545, at 4305932608 00000100:00000010:0.0:1786176585.740486:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290ccc0. 00010000:00000001:0.0:1786176585.804468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176585.804474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176585.804482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176585.804486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176585.804494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176585.804497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176586.828474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176586.828479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176586.828485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176586.828486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176586.828490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176586.828492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176586.828547:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176586.828551:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176586.828554:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176586.956525:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176586.956536:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176586.956539:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176586.956542:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176586.956547:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176586.956555:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11261 00000400:00000200:3.0:1786176586.956565:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176586.956572:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176586.956575:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176586.956578:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956582:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176586.956584:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956585:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176586.956589:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176586.956590:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176586.956593:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176586.956596:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176586.956597:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176586.956600:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176586.956603:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956605:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176586.956607:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956609:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176586.956612:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176586.956616:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956619:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176586.956622:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176586.956625:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176586.956629:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.956631:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176586.956633:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176586.956643:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786176586.956648:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176586.956652:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176586.956654:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.956657:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176586.956659:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176586.956663:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176586.956670:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176586.956675:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176586.956685:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235511040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176586.956696:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176586.956701:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176586.956706:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235511040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176586.956735:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176586.956738:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176586.956741:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176586.956742:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176586.956743:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176586.956746:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176586.956748:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11261 00000400:00000200:3.0:1786176586.956751:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176586.956754:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176586.956755:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176586.956757:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956759:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176586.956760:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956762:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176586.956764:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176586.956765:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176586.956766:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176586.956767:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176586.956768:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176586.956770:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176586.956772:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956773:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176586.956774:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956776:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176586.956777:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176586.956779:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956781:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176586.956782:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176586.956787:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176586.956788:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.956790:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176586.956794:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176586.956796:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.956798:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176586.956802:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786176586.956804:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176586.956809:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176586.956811:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.956814:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176586.956816:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176586.956819:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176586.956823:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176586.956827:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176586.956832:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235511168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176586.956844:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176586.956849:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176586.956854:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235511168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176586.956866:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176586.956870:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176586.956877:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176586.956880:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176586.956883:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176586.956888:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176586.956893:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11261 00000400:00000200:3.0:1786176586.956898:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176586.956903:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176586.956905:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176586.956908:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956912:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176586.956916:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956919:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176586.956924:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176586.956928:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176586.956931:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176586.956934:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176586.956939:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176586.956943:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176586.956951:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956956:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176586.956963:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956969:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176586.956975:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176586.956980:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176586.956984:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176586.956986:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176586.956990:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176586.956993:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.956997:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176586.956999:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176586.957001:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.957005:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176586.957010:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786176586.957012:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176586.957016:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176586.957019:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.957021:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176586.957022:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176586.957023:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176586.957026:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176586.957029:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176586.957034:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235511296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176586.957043:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176586.957047:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176586.957054:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235511296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176586.957072:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176586.957075:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176586.957078:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176586.957079:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176586.957082:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176586.957087:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176586.957091:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11261 00000400:00000200:3.0:1786176586.957096:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176586.957104:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176586.957107:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176586.957114:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.957118:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176586.957122:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.957125:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176586.957127:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176586.957130:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176586.957132:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176586.957133:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176586.957135:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176586.957139:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176586.957145:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.957147:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176586.957150:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.957153:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176586.957155:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176586.957160:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:2.1:1786176586.957160:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176586.957163:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176586.957167:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1786176586.957169:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 02000000:00000001:3.0:1786176586.957170:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176586.957173:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.957176:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176586.957178:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176586.957183:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.957187:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176586.957191:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176586.957195:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176586.957196:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176586.957198:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.957199:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176586.957202:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176586.957205:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176586.957210:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176586.957214:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176586.957219:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235511424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176586.957242:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176586.957245:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176586.957249:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235511424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176586.957258:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176586.957263:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176586.957270:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176586.957273:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176586.957275:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176586.957279:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176586.957284:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11261 00000400:00000200:3.0:1786176586.957289:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176586.957294:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176586.957297:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176586.957301:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.957305:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176586.957309:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.957313:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176586.957317:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176586.957320:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176586.957322:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176586.957324:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176586.957328:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176586.957332:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176586.957334:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.957336:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176586.957338:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176586.957343:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176586.957345:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176586.957350:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176586.957385:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176586.957389:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176586.957395:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176586.957398:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.957403:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176586.957406:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176586.957411:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.957415:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176586.957419:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176586.957422:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176586.957428:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176586.957429:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176586.957431:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176586.957433:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176586.957436:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176586.957441:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176586.957446:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176586.957454:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235511552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176586.957464:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176586.957466:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176586.957471:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235511552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176586.957489:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176586.957494:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11271) 00000800:00000200:2.1:1786176586.957909:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176586.957912:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176586.958219:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176586.958222:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176586.958574:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176586.958577:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176586.958845:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176586.958847:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00010000:00000001:0.0:1786176587.852474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176587.852480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176587.852489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176587.852494:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176587.852498:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176587.852501:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176587.852505:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176587.852508:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176587.852510:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176587.852512:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176587.852517:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176587.852518:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176588.876548:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176588.876552:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176588.876556:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176588.876558:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176588.876561:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176588.876563:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176588.876611:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176588.876615:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176588.876618:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176589.900479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176589.900485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176589.900491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176589.900494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176589.900498:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176589.900500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176590.540519:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176590.540526:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176590.540527:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176590.540538:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176590.540541:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176590.540544:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176590.540547:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176590.540549:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176590.540551:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176590.860541:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305937664, at 4305937728 00000100:00000010:0.0:1786176590.860569:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cd80. 00010000:00000001:0.0:1786176590.924480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176590.924487:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176590.924492:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176590.924497:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176590.924502:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176590.924506:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176590.924510:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176590.924517:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176590.924524:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176590.924528:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176590.924535:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176590.924537:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176590.924539:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176590.924540:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176590.924542:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176590.924544:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176590.924546:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176590.924548:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176590.924602:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176590.924605:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176590.924608:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176591.564493:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176591.564504:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176591.564507:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176591.564514:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176591.564519:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176591.564523:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176591.564526:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176591.564529:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176591.564534:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176591.948472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176591.948479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176591.948487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176591.948491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176591.948497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176591.948501:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176592.076478:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176592.076490:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176592.076493:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176592.076497:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176592.076502:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176592.076509:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11266 00000400:00000200:3.0:1786176592.076518:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176592.076525:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176592.076528:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176592.076531:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.076535:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176592.076537:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.076540:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176592.076544:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176592.076546:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176592.076549:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176592.076553:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176592.076555:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176592.076558:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176592.076561:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.076563:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176592.076565:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.076567:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176592.076570:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176592.076573:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176592.076575:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176592.076578:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176592.076581:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176592.076584:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.076586:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176592.076590:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176592.076601:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176592.076605:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176592.076610:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176592.076612:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.076615:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176592.076618:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176592.076622:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176592.076629:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176592.076636:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176592.076645:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235511680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176592.076656:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176592.076661:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176592.076665:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235511680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176592.076691:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176592.076694:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176592.076697:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176592.076704:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176592.076706:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176592.076708:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176592.076711:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11266 00000400:00000200:3.0:1786176592.076715:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176592.076720:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176592.076723:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176592.076727:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.076730:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176592.076733:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.076735:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176592.076737:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176592.076740:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176592.076744:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176592.076748:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176592.076752:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176592.076762:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176592.076772:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.076786:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176592.076792:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.076797:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176592.076802:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176592.076809:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176592.076815:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176592.076819:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176592.076828:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176592.076834:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.076840:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176592.076847:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176592.076855:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.076859:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176592.076866:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176592.076871:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176592.076876:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176592.076877:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.076880:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176592.076883:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176592.076886:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176592.076892:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176592.076900:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176592.076911:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235511808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176592.076938:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176592.076944:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176592.076955:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235511808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176592.076979:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176592.076983:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176592.076990:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176592.076995:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176592.076997:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176592.077002:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176592.077008:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11266 00000400:00000200:3.0:1786176592.077042:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176592.077052:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176592.077058:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176592.077065:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077072:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176592.077075:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077080:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176592.077082:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176592.077086:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176592.077089:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176592.077094:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176592.077098:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176592.077103:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000200:2.1:1786176592.077106:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786176592.077110:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:2.1:1786176592.077112:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1786176592.077115:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176592.077123:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077128:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176592.077134:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176592.077139:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077144:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176592.077150:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176592.077155:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176592.077160:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.077165:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176592.077170:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176592.077175:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.077192:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176592.077198:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176592.077204:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176592.077209:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176592.077212:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.077215:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176592.077218:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176592.077221:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176592.077228:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176592.077235:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176592.077244:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235511936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176592.077258:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176592.077261:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176592.077265:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235511936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176592.077279:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176592.077285:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176592.077290:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176592.077294:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176592.077297:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176592.077301:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176592.077305:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11266 00000400:00000200:3.0:1786176592.077314:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176592.077320:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176592.077325:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176592.077333:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077338:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176592.077345:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077351:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176592.077395:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176592.077397:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176592.077401:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176592.077406:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176592.077410:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176592.077416:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176592.077421:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077425:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176592.077431:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077437:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176592.077443:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176592.077452:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077459:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176592.077464:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176592.077469:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176592.077476:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.077482:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176592.077488:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176592.077496:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.077503:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176592.077510:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176592.077517:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176592.077524:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176592.077529:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.077533:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176592.077537:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176592.077541:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176592.077548:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176592.077555:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176592.077568:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235512064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176592.077587:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176592.077594:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176592.077606:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235512064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176592.077619:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176592.077626:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176592.077635:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176592.077639:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176592.077643:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176592.077647:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176592.077655:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11266 00000400:00000200:3.0:1786176592.077666:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176592.077675:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176592.077681:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176592.077688:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077692:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176592.077699:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077707:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176592.077713:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176592.077716:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176592.077719:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176592.077724:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176592.077731:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176592.077737:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176592.077747:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077750:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176592.077756:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077761:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176592.077768:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176592.077776:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176592.077783:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176592.077787:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176592.077793:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176592.077801:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.077806:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176592.077810:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176592.077816:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.077821:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176592.077828:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176592.077834:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176592.077841:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176592.077846:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176592.077850:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176592.077854:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176592.077858:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176592.077866:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176592.077875:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176592.077887:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235512192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176592.077903:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176592.077909:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176592.077920:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235512192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176592.077939:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176592.077948:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11276) 00000800:00000200:2.1:1786176592.078011:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176592.078015:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786176592.078383:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176592.078386:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786176592.078697:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176592.078700:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786176592.079130:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176592.079135:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1786176592.588477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176592.588486:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176592.588489:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176592.588495:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176592.588500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176592.588504:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176592.588507:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176592.588511:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176592.588514:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176592.972484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176592.972491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176592.972502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176592.972507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176592.972515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176592.972521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176592.972583:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176592.972586:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176592.972589:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176593.612474:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176593.612482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176593.612484:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176593.612490:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176593.612494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176593.612497:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176593.612500:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176593.612503:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176593.612506:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176593.996471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176593.996475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176593.996481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176593.996482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176593.996485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176593.996487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176594.636490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176594.636500:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176594.636503:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176594.636510:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176594.636516:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176594.636520:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176594.636524:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176594.636527:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176594.636531:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176595.020445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176595.020452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176595.020459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176595.020463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176595.020469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176595.020473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176595.020527:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176595.020531:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176595.020535:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176595.660564:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176595.660571:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176595.660573:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176595.660578:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176595.660582:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176595.660586:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176595.660588:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176595.660591:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176595.660594:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176595.980473:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305942784, at 4305942848 00000100:00000010:0.0:1786176595.980499:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c720. 00010000:00000001:0.0:1786176596.044470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176596.044475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176596.044484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176596.044487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176596.044491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176596.044494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176596.684470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176596.684479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176596.684481:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176596.684487:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176596.684493:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176596.684497:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176596.684501:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176596.684504:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176596.684507:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176597.068494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176597.068498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176597.068512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176597.068518:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176597.068525:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176597.068529:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176597.068592:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176597.068597:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176597.068600:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176597.197166:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176597.197172:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176597.197980:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176597.197983:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176597.198318:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176597.198321:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176597.198649:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176597.198652:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176597.198959:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176597.198962:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00000001:3.0:1786176597.708484:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176597.708494:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176597.708497:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176597.708513:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176597.708522:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176597.708526:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176597.708531:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176597.708534:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176597.708538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176598.092465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176598.092470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176598.092476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176598.092480:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176598.092484:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176598.092488:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176598.092495:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176598.092498:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176598.092500:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176598.092504:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176598.092512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176598.092514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176598.732468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176598.732479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176598.732481:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176598.732488:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176598.732493:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176598.732501:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176598.732505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176598.732508:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176598.732513:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176599.116493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176599.116498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176599.116505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176599.116508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176599.116511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176599.116514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176599.116574:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176599.116579:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176599.116583:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176600.140493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176600.140500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176600.140510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176600.140514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176600.140519:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176600.140524:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176601.100516:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305947904, at 4305947968 00000100:00000010:0.0:1786176601.100562:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cae0. 00010000:00000001:0.0:1786176601.164509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176601.164530:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176601.164535:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176601.164541:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176601.164552:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176601.164557:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176601.164561:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176601.164568:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176601.164574:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176601.164577:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176601.164588:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176601.164591:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176601.164592:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176601.164595:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176601.164597:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176601.164598:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176601.164599:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176601.164601:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176601.164679:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176601.164686:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176601.164690:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176602.188527:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176602.188535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176602.188548:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176602.188551:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176602.188556:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176602.188560:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176602.316484:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176602.316497:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176602.316500:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176602.316512:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176602.316516:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176602.316524:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11276 00000400:00000200:3.0:1786176602.316534:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176602.316540:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176602.316544:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176602.316546:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316549:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176602.316551:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316553:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176602.316556:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176602.316557:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176602.316559:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176602.316562:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176602.316564:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176602.316565:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176602.316568:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316569:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176602.316571:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316572:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176602.316574:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176602.316577:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316578:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176602.316580:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176602.316583:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176602.316585:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.316586:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176602.316588:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176602.316597:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176602.316600:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176602.316603:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176602.316605:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.316608:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176602.316610:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176602.316614:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176602.316620:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176602.316626:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176602.316634:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235512960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176602.316642:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176602.316646:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176602.316649:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235512960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176602.316668:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176602.316671:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176602.316674:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176602.316675:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176602.316676:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176602.316678:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176602.316679:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11276 00000400:00000200:3.0:1786176602.316682:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176602.316684:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176602.316685:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176602.316687:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316689:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176602.316690:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316692:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176602.316694:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176602.316695:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176602.316696:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176602.316699:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176602.316701:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176602.316703:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176602.316705:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316706:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176602.316709:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316711:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176602.316715:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176602.316719:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316723:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176602.316725:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176602.316731:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176602.316737:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.316741:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176602.316746:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176602.316753:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.316758:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176602.316762:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176602.316768:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176602.316771:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176602.316775:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.316779:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176602.316781:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176602.316785:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176602.316788:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176602.316793:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176602.316802:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235513088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176602.316814:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176602.316818:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176602.316823:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235513088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176602.316839:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176602.316844:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176602.316848:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176602.316851:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176602.316853:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176602.316859:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176602.316864:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11276 00000400:00000200:3.0:1786176602.316870:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176602.316877:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176602.316880:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176602.316884:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316888:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176602.316890:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316895:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176602.316898:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176602.316899:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176602.316899:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176602.316901:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176602.316902:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176602.316903:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176602.316907:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316909:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176602.316910:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316912:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176602.316916:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176602.316921:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176602.316923:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176602.316926:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176602.316931:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176602.316933:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.316936:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176602.316939:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176602.316942:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.316945:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176602.316950:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176602.316954:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176602.316959:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176602.316963:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.316968:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176602.316970:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176602.316971:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176602.316975:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176602.316979:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176602.316986:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235513216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176602.316994:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176602.316996:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176602.317002:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235513216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176602.317014:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176602.317016:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176602.317021:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176602.317022:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176602.317025:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1786176602.317026:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176602.317029:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:2.1:1786176602.317031:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000100:00080000:3.0:1786176602.317034:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11276 00000400:00000200:3.0:1786176602.317041:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176602.317048:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176602.317051:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176602.317054:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.317058:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176602.317064:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.317067:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176602.317068:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176602.317072:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176602.317076:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176602.317078:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176602.317096:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176602.317102:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176602.317111:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.317115:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176602.317119:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.317123:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176602.317129:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176602.317137:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176602.317142:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176602.317148:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176602.317153:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176602.317158:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.317164:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176602.317167:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176602.317173:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.317175:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176602.317179:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176602.317183:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176602.317189:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176602.317192:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.317195:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176602.317198:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176602.317202:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176602.317208:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176602.317214:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176602.317223:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235513344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176602.317244:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176602.317249:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176602.317256:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235513344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176602.317272:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176602.317276:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176602.317279:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176602.317282:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176602.317286:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176602.317289:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176602.317294:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11276 00000400:00000200:3.0:1786176602.317302:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176602.317310:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176602.317312:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176602.317319:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.317322:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176602.317327:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.317331:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176602.317334:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176602.317338:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176602.317341:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176602.317344:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176602.317350:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176602.317355:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176602.317384:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.317389:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176602.317405:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176602.317412:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176602.317418:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176602.317423:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176602.317429:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176602.317435:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176602.317440:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176602.317445:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.317447:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176602.317450:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176602.317454:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.317456:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176602.317471:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176602.317476:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176602.317480:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176602.317483:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176602.317488:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176602.317493:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176602.317499:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176602.317508:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176602.317516:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176602.317528:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235513472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176602.317544:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176602.317547:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176602.317558:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235513472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176602.317577:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176602.317582:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11286) 00000800:00000200:2.1:1786176602.317676:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176602.317678:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176602.317878:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176602.317879:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176602.318072:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176602.318074:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176602.318352:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176602.318354:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00010000:00000001:0.0:1786176603.212461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176603.212466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176603.212473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176603.212478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176603.212489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176603.212492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176603.212557:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176603.212562:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176603.212565:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176604.236462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176604.236467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176604.236474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176604.236476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176604.236478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176604.236479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176605.260485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176605.260491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176605.260498:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176605.260500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176605.260503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176605.260507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176605.260563:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176605.260568:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176605.260573:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176606.220467:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305953024, at 4305953088 00000100:00000010:0.0:1786176606.220485:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cb40. 00010000:00000001:0.0:1786176606.284471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176606.284475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176606.284482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176606.284485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176606.284489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176606.284493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176607.308471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176607.308476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176607.308481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176607.308484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176607.308490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176607.308491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176607.308562:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176607.308567:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176607.308572:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176607.436561:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176607.436573:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176607.436575:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176607.436580:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176607.436585:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176607.436593:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11281 00000400:00000200:0.0:1786176607.436602:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176607.436608:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176607.436611:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176607.436614:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.436618:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176607.436620:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.436622:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176607.436626:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176607.436628:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176607.436630:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176607.436633:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176607.436635:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176607.436637:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176607.436639:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.436641:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176607.436643:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.436644:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176607.436647:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176607.436650:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176607.436651:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176607.436653:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176607.436657:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176607.436659:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.436661:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176607.436663:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176607.436674:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786176607.436677:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176607.436681:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176607.436682:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.436693:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176607.436696:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176607.436701:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176607.436712:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786176607.436718:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176607.436728:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235513600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176607.436738:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176607.436742:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176607.436747:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235513600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176607.436767:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176607.436774:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176607.436777:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176607.436778:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176607.436780:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176607.436783:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176607.436785:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11281 00000400:00000200:0.0:1786176607.436787:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176607.436789:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176607.436792:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176607.436794:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.436796:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176607.436799:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.436801:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176607.436803:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176607.436804:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176607.436805:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176607.436807:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176607.436808:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176607.436810:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176607.436812:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.436815:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176607.436818:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.436820:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176607.436825:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176607.436830:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176607.436832:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176607.436834:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176607.436841:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176607.436845:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.436850:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176607.436856:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176607.436863:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.436870:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176607.436876:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786176607.436880:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176607.436887:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176607.436891:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.436895:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176607.436898:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176607.436903:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176607.436911:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176607.436917:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176607.436926:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235513728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176607.436935:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176607.436938:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176607.436944:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235513728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176607.436951:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176607.436953:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176607.436959:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176607.436963:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176607.436966:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176607.436969:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176607.436975:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11281 00000400:00000200:0.0:1786176607.436983:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176607.436989:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176607.436992:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176607.436998:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437002:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176607.437007:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437010:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176607.437013:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176607.437015:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176607.437017:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176607.437020:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176607.437024:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176607.437026:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176607.437029:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437044:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176607.437046:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437048:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176607.437049:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176607.437051:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437053:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176607.437054:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176607.437056:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176607.437059:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.437063:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176607.437065:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176607.437068:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.437070:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176607.437076:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786176607.437080:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176607.437083:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176607.437086:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.437089:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176607.437091:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176607.437093:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176607.437099:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176607.437104:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176607.437110:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235513856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176607.437122:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176607.437124:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176607.437131:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235513856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176607.437148:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176607.437152:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176607.437155:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176607.437157:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176607.437160:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176607.437164:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176607.437169:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11281 00000400:00000200:0.0:1786176607.437173:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176607.437177:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176607.437179:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176607.437183:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437187:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1786176607.437190:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:0.0:1786176607.437191:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437197:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:2.1:1786176607.437198:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000020:00000001:0.0:1786176607.437205:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176607.437208:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176607.437210:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176607.437212:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176607.437214:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176607.437219:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176607.437224:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437228:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176607.437230:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437232:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176607.437236:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176607.437240:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437244:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176607.437247:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176607.437250:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176607.437255:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.437259:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176607.437262:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176607.437265:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.437267:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176607.437272:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786176607.437277:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176607.437281:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176607.437283:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.437287:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176607.437289:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176607.437292:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176607.437296:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176607.437299:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176607.437313:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235513984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176607.437323:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176607.437325:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176607.437328:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235513984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176607.437334:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176607.437339:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176607.437341:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176607.437342:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176607.437343:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176607.437346:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176607.437350:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11281 00000400:00000200:0.0:1786176607.437380:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176607.437384:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176607.437386:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176607.437390:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437395:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176607.437398:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437404:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176607.437407:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176607.437410:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176607.437413:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176607.437414:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176607.437417:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176607.437421:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176607.437429:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437434:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176607.437437:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437439:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176607.437442:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176607.437447:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176607.437450:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176607.437455:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176607.437457:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176607.437461:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.437463:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176607.437467:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176607.437469:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.437471:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176607.437475:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786176607.437479:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176607.437484:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176607.437486:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176607.437491:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176607.437495:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176607.437498:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176607.437505:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176607.437512:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176607.437523:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235514112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176607.437539:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176607.437544:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176607.437552:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235514112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176607.437562:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176607.437567:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11292) 00000800:00000200:2.1:1786176607.437829:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176607.437833:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176607.438148:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176607.438151:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176607.438402:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176607.438404:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176607.438636:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176607.438638:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00010000:00000001:0.0:1786176608.332505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176608.332510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176608.332520:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176608.332525:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176608.332528:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176608.332534:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176608.332538:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176608.332542:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176608.332547:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176608.332549:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176608.332554:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176608.332557:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176609.356511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176609.356525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176609.356534:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176609.356538:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176609.356543:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176609.356546:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176609.356615:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176609.356619:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176609.356630:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176610.380458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176610.380465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176610.380471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176610.380473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176610.380480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176610.380482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176611.340471:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305958144, at 4305958208 00000100:00000010:0.0:1786176611.340486:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290ce40. 00010000:00000001:0.0:1786176611.404465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176611.404472:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176611.404476:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176611.404481:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176611.404485:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176611.404489:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176611.404493:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176611.404495:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176611.404501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176611.404503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176611.404506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176611.404509:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176611.404511:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176611.404512:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176611.404514:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176611.404515:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176611.404516:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176611.404518:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176611.404586:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176611.404591:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176611.404594:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176612.428431:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176612.428439:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176612.428447:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176612.428449:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176612.428456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176612.428459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176612.557204:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176612.557211:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176612.557855:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176612.557858:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176612.558189:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176612.558192:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176612.558552:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176612.558555:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176612.558879:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176612.558882:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1786176613.452463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176613.452467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176613.452489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176613.452493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176613.452498:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176613.452500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176613.452558:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176613.452563:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176613.452566:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176614.476484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176614.476491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176614.476501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176614.476505:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176614.476513:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176614.476518:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176615.500483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176615.500487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176615.500495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176615.500498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176615.500505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176615.500508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176615.500578:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176615.500584:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176615.500587:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176616.460453:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305963265, at 4305963328 00000100:00000010:0.0:1786176616.460467:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c000. 00010000:00000001:0.0:1786176616.524471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176616.524475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176616.524484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176616.524486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176616.524490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176616.524492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176617.548467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176617.548472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176617.548477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176617.548480:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176617.548483:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176617.548486:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176617.548490:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176617.548493:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176617.548497:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176617.548503:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176617.548512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176617.548515:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176617.548572:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176617.548577:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176617.548580:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176617.677005:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176617.677010:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176617.677631:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176617.677634:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176617.677870:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176617.677872:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176617.678093:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176617.678095:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786176617.678396:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176617.678398:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1786176618.572469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176618.572473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176618.572480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176618.572483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176618.572486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176618.572489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176619.596472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176619.596477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176619.596481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176619.596483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176619.596490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176619.596491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176619.596582:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176619.596585:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176619.596588:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176620.620514:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176620.620521:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176620.620525:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176620.620535:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176620.620540:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176620.620543:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176620.620546:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176620.620550:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176620.620553:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176620.620556:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176620.620561:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176620.620565:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176620.620580:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176620.620583:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176620.620586:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176620.620587:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176620.620589:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176620.620591:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1786176621.196484:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:3.0:1786176621.196494:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:3.0:1786176621.196500:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1786176621.196671:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000040:0.0:1786176621.580450:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305968384, at 4305968448 00000100:00000010:0.0:1786176621.580465:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c300. 00010000:00000001:0.0:1786176621.644467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176621.644472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176621.644477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176621.644480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176621.644485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176621.644486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176621.644545:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176621.644548:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176621.644551:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176622.668477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176622.668484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176622.668494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176622.668498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176622.668505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176622.668509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176622.797251:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176622.797259:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176622.797833:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176622.797835:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176622.798166:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176622.798169:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176622.798490:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176622.798493:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176622.798792:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176622.798795:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1786176623.692464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176623.692470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176623.692475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176623.692476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176623.692479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176623.692481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176623.692543:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176623.692548:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176623.692552:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176624.716471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176624.716477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176624.716483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176624.716485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176624.716489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176624.716491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176625.740470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176625.740475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176625.740479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176625.740481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176625.740486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176625.740489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176625.740552:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176625.740555:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176625.740558:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176626.700472:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305973505, at 4305973568 00000100:00000010:0.0:1786176626.700488:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c9c0. 00010000:00000001:0.0:1786176626.764473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176626.764478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176626.764484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176626.764487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176626.764490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176626.764492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176627.788438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176627.788448:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176627.788457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176627.788462:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176627.788467:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176627.788472:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176627.788478:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176627.788482:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176627.788486:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176627.788488:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176627.788494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176627.788495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176627.788544:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176627.788548:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176627.788551:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176627.916512:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176627.916522:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176627.916524:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176627.916527:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176627.916531:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176627.916537:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11302 00000400:00000200:3.0:1786176627.916545:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176627.916549:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176627.916552:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176627.916554:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916557:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176627.916559:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916560:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176627.916564:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176627.916566:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176627.916568:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176627.916571:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176627.916572:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176627.916574:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176627.916576:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916577:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176627.916579:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916580:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176627.916583:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176627.916585:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916587:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176627.916588:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176627.916591:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176627.916593:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.916595:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176627.916597:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176627.916606:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786176627.916609:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176627.916612:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176627.916613:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.916616:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176627.916618:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176627.916621:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176627.916627:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176627.916633:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176627.916640:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235516160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176627.916648:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176627.916652:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176627.916655:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235516160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176627.916675:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176627.916677:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176627.916680:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176627.916681:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176627.916682:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176627.916688:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176627.916690:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11302 00000400:00000200:3.0:1786176627.916692:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176627.916694:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176627.916696:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176627.916698:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916699:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176627.916701:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916702:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176627.916704:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176627.916705:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176627.916706:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176627.916708:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176627.916709:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176627.916710:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176627.916712:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916714:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176627.916715:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916717:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176627.916718:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176627.916720:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916722:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176627.916723:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176627.916728:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176627.916729:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.916731:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176627.916737:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176627.916741:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.916743:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176627.916747:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786176627.916749:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176627.916752:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176627.916754:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.916755:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176627.916758:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176627.916760:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176627.916764:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176627.916768:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176627.916772:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235516288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176627.916780:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176627.916783:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176627.916787:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235516288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176627.916799:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176627.916803:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176627.916806:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176627.916808:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176627.916809:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176627.916811:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176627.916814:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11302 00000400:00000200:3.0:1786176627.916818:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176627.916822:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176627.916824:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176627.916829:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916846:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176627.916849:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916850:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176627.916852:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176627.916853:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176627.916854:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176627.916855:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176627.916857:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176627.916858:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176627.916860:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916861:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176627.916862:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916864:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176627.916869:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176627.916872:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916877:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176627.916878:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176627.916880:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176627.916884:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.916887:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176627.916890:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176627.916892:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.916895:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176627.916900:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786176627.916904:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176627.916907:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176627.916909:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.916911:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176627.916912:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176627.916913:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176627.916916:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176627.916920:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176627.916925:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235516416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176627.916934:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176627.916936:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176627.916942:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235516416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176627.916948:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176627.916952:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176627.916956:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176627.916961:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176627.916963:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176627.916968:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176627.916974:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11302 00000400:00000200:3.0:1786176627.916978:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176627.916984:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176627.916987:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176627.916992:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.916995:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176627.916999:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.917001:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176627.917004:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176627.917006:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176627.917008:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176627.917011:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176627.917013:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176627.917015:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176627.917021:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.917022:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176627.917026:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.917029:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176627.917031:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176627.917033:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176627.917035:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176627.917039:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176627.917042:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176627.917044:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.917046:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176627.917047:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176627.917057:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.917061:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176627.917070:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176627.917075:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176627.917077:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176627.917080:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.917084:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176627.917085:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176627.917087:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176627.917091:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176627.917094:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176627.917100:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235516544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176627.917110:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176627.917113:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176627.917119:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235516544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176627.917127:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176627.917129:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176627.917133:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176627.917135:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176627.917137:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176627.917142:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176627.917147:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11302 00000400:00000200:3.0:1786176627.917151:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176627.917155:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176627.917157:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176627.917162:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.917167:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176627.917168:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.917169:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176627.917171:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176627.917174:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176627.917176:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176627.917177:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176627.917179:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176627.917182:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176627.917187:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.917191:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176627.917194:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176627.917197:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176627.917200:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176627.917204:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176627.917208:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176627.917209:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786176627.917210:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176627.917214:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.1:1786176627.917216:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 02000000:00000001:3.0:1786176627.917217:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.917220:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176627.917222:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176627.917226:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.917230:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176627.917237:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176627.917241:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176627.917244:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176627.917255:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176627.917259:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176627.917260:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176627.917262:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176627.917267:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176627.917270:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176627.917279:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235516672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176627.917293:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176627.917298:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176627.917307:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235516672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176627.917326:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176627.917332:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11312) 00000800:00000200:2.1:1786176627.917773:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176627.917776:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176627.918115:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176627.918117:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176627.918441:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176627.918443:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786176627.918720:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176627.918723:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1786176628.812467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176628.812472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176628.812478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176628.812482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176628.812486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176628.812488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176629.836507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176629.836510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176629.836514:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176629.836515:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176629.836517:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176629.836518:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176629.836556:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176629.836560:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176629.836563:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176630.860480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176630.860488:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176630.860492:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176630.860499:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176630.860504:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176630.860509:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176630.860513:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176630.860518:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176630.860523:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176630.860525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176630.860529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176630.860531:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176630.860532:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176630.860534:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176630.860536:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176630.860537:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176630.860540:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176630.860542:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176631.820449:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305978624, at 4305978688 00000100:00000010:0.0:1786176631.820468:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cb40. 00010000:00000001:0.0:1786176631.884480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176631.884485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176631.884495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176631.884499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176631.884505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176631.884509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176631.884566:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176631.884572:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176631.884576:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176632.524550:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176632.524563:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176632.524567:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176632.524575:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176632.524580:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176632.524586:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176632.524590:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176632.524594:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176632.524599:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176632.908611:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176632.908626:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176632.908631:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176632.908633:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176632.908636:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176632.908638:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176633.037074:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176633.037080:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176633.037763:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176633.037765:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176633.038026:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176633.038029:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176633.038281:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176633.038283:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176633.038653:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176633.038656:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000001:3.0:1786176633.548477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176633.548485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176633.548486:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176633.548491:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176633.548494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176633.548498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176633.548500:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176633.548503:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176633.548505:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176633.932446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176633.932452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176633.932461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176633.932467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176633.932475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176633.932478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176633.932543:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176633.932548:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176633.932552:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176634.572489:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176634.572497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176634.572500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176634.572506:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176634.572510:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176634.572514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176634.572517:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176634.572520:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176634.572524:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176634.956458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176634.956464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176634.956471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176634.956474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176634.956480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176634.956482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176635.596465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176635.596474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176635.596477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176635.596483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176635.596488:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176635.596492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176635.596494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176635.596497:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176635.596501:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176635.980458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176635.980463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176635.980469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176635.980472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176635.980478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176635.980481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176635.980538:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176635.980543:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176635.980546:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176636.620422:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176636.620430:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176636.620431:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176636.620436:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176636.620440:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176636.620443:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176636.620445:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176636.620448:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176636.620450:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176636.940451:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305983744, at 4305983808 00000100:00000010:0.0:1786176636.940471:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62060. 00010000:00000001:0.0:1786176637.004463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176637.004467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176637.004472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176637.004475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176637.004482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176637.004485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176637.644442:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176637.644449:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176637.644451:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176637.644456:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176637.644460:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176637.644464:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176637.644467:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176637.644469:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176637.644471:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176638.028473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176638.028478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176638.028486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176638.028492:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176638.028497:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176638.028500:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176638.028506:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176638.028510:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176638.028515:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176638.028521:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176638.028530:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176638.028532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176638.028584:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176638.028588:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176638.028592:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176638.156993:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176638.157350:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176638.157673:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176638.157850:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176638.157999:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176638.668459:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176638.668468:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176638.668471:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176638.668478:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176638.668483:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176638.668487:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176638.668490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176638.668493:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176638.668496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176639.052483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176639.052489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176639.052498:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176639.052503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176639.052512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176639.052516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176639.692451:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176639.692460:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176639.692463:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176639.692469:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176639.692473:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176639.692477:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176639.692481:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176639.692483:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176639.692487:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176640.076435:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176640.076440:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176640.076446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176640.076448:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176640.076454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176640.076457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176640.076507:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176640.076511:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176640.076514:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176641.100476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176641.100481:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176641.100484:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176641.100487:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176641.100491:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176641.100493:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176641.100498:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176641.100500:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176641.100505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176641.100507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176641.100512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176641.100515:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176641.100517:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176641.100519:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176641.100522:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176641.100524:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176641.100529:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176641.100531:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:3.0:1786176641.228673:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:3.0:1786176641.228744:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1786176641.228754:0:2383:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176641.228758:0:2383:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:3.0:1786176641.228765:0:2383:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176641.228769:0:2383:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1786176641.228774:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176641.228779:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:3.0:1786176641.228784:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176641.228789:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176641.228796:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ebb0 00000400:00000010:3.0:1786176641.228799:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ebb0. 00000100:00000001:3.0:1786176641.228804:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176641.228819:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735c00 x1872930235517440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176641.228843:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176641.228849:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235517440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176641.228857:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176641.228861:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176641.228873:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176641.228877:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:3.0:1786176641.228884:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176641.228890:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176641.228895:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e330 00000400:00000010:3.0:1786176641.228900:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e330. 00000100:00000001:3.0:1786176641.228905:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176641.228914:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735880 x1872930235517568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176641.228929:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176641.228935:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735880 x1872930235517568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176641.228950:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176641.228955:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176641.228966:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176641.228971:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:3.0:1786176641.228978:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176641.228986:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176641.228992:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e6e8 00000400:00000010:3.0:1786176641.228996:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e6e8. 00000100:00000001:3.0:1786176641.229002:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176641.229010:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31734700 x1872930235517696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176641.229026:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176641.229035:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734700 x1872930235517696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176641.229056:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176641.229059:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176641.229068:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176641.229076:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:3.0:1786176641.229082:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176641.229092:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176641.229098:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eaa0 00000400:00000010:3.0:1786176641.229103:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eaa0. 00000100:00000001:3.0:1786176641.229108:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176641.229117:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31737b80 x1872930235517824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176641.229133:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176641.229138:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235517824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176641.229144:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176641.229148:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176641.229156:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176641.229163:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:3.0:1786176641.229168:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176641.229178:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176641.229184:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ee58 00000400:00000010:3.0:1786176641.229189:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ee58. 00000100:00000001:3.0:1786176641.229200:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176641.229208:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735500 x1872930235517952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176641.229226:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176641.229232:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235517952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176663 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176641.229245:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176641.229249:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1786176642.060464:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305988864, at 4305988928 00000100:00000010:0.0:1786176642.060478:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62900. 00010000:00000001:0.0:1786176642.124450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176642.124458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176642.124467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176642.124471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176642.124478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176642.124481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176642.124567:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176642.124571:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176642.124574:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176643.148470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176643.148476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176643.148487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176643.148489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176643.148494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176643.148497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176643.276543:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176643.276552:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176643.276554:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176643.276557:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176643.276560:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176643.276565:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11317 00000400:00000200:3.0:1786176643.276572:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176643.276577:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176643.276579:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176643.276582:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.276585:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176643.276587:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.276588:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176643.276592:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176643.276593:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176643.276595:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176643.276598:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176643.276599:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176643.276601:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176643.276603:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.276604:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176643.276606:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.276607:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176643.276609:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176643.276612:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176643.276613:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176643.276615:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176643.276618:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176643.276620:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.276621:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176643.276623:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176643.276628:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176643.276631:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176643.276634:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176643.276635:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.276638:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176643.276640:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176643.276644:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176643.276650:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176643.276654:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176643.276661:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235518080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176643.276668:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176643.276672:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176643.276676:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235518080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176643.276708:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176643.276711:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176643.276714:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176643.276715:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176643.276716:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176643.276718:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176643.276719:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11317 00000400:00000200:3.0:1786176643.276722:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176643.276724:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176643.276726:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176643.276728:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.276729:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176643.276731:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.276732:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176643.276734:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176643.276736:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176643.276738:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176643.276740:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176643.276742:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176643.276746:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176643.276750:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.276759:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176643.276764:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.276768:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176643.276774:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176643.276781:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176643.276788:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176643.276793:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176643.276800:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176643.276817:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.276825:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176643.276832:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176643.276839:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.276844:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176643.276850:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176643.276855:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176643.276860:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176643.276863:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.276866:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176643.276868:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176643.276873:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176643.276881:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176643.276887:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176643.276897:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235518208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176643.276911:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176643.276916:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176643.276940:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235518208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176643.276958:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176643.276963:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176643.276971:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176643.276974:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176643.276976:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176643.276981:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176643.276987:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11317 00000400:00000200:3.0:1786176643.276996:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176643.277004:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176643.277009:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176643.277017:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277024:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176643.277030:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277035:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176643.277040:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176643.277044:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176643.277048:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176643.277053:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176643.277056:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176643.277060:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176643.277068:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277073:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176643.277079:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277083:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176643.277086:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176643.277093:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277096:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176643.277100:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176643.277105:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176643.277110:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.277114:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:1.1:1786176643.277116:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000008:00100000:3.0:1786176643.277118:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176643.277123:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176643.277123:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786176643.277127:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176643.277133:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176643.277139:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176643.277145:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176643.277147:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.277153:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176643.277157:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176643.277161:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176643.277169:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176643.277175:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176643.277192:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235518336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176643.277205:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176643.277210:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176643.277220:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235518336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176643.277231:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176643.277236:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176643.277243:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176643.277246:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176643.277249:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176643.277254:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176643.277258:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11317 00000400:00000200:3.0:1786176643.277265:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176643.277271:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176643.277276:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176643.277284:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277290:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176643.277294:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277299:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176643.277305:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176643.277308:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176643.277312:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176643.277316:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176643.277481:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176643.277487:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176643.277496:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277501:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176643.277508:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277513:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176643.277518:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176643.277524:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277530:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176643.277536:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176643.277542:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176643.277547:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.277552:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176643.277556:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176643.277563:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.277569:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176643.277575:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176643.277580:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176643.277584:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176643.277587:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.277592:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176643.277595:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176643.277599:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176643.277608:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176643.277614:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176643.277626:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235518464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176643.277654:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176643.277660:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176643.277669:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235518464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176643.277699:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176643.277706:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176643.277712:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176643.277716:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176643.277720:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176643.277724:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176643.277731:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11317 00000400:00000200:3.0:1786176643.277742:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176643.277750:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176643.277756:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176643.277762:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277767:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176643.277772:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277777:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176643.277782:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176643.277785:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176643.277789:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176643.277792:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176643.277797:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176643.277802:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176643.277810:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277813:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176643.277819:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277824:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176643.277829:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176643.277838:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176643.277843:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176643.277846:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176643.277852:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176643.277858:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.277864:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176643.277874:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176643.277880:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.277884:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176643.277891:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176643.277896:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176643.277901:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176643.277904:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176643.277910:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176643.277914:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1786176643.277915:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786176643.277919:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176643.277920:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 02000000:00000010:3.0:1786176643.277927:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176643.277934:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176643.277944:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235518592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176643.277961:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176643.277966:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176643.277975:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235518592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176643.278008:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176643.278018:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11327) 00000800:00000200:1.1:1786176643.278251:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176643.278254:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176643.278739:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176643.278743:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176643.279046:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176643.279047:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1786176644.172505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176644.172511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176644.172518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176644.172521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176644.172527:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176644.172531:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176644.172638:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176644.172644:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176644.172649:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176645.196429:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176645.196435:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176645.196443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176645.196446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176645.196452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176645.196455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176646.220455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176646.220460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176646.220468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176646.220471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176646.220474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176646.220475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176646.220524:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176646.220527:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176646.220531:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176646.860497:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176646.860506:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176646.860509:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176646.860515:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176646.860521:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176646.860526:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176646.860529:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176646.860532:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176646.860536:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176647.180456:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305993984, at 4305994048 00000100:00000010:0.0:1786176647.180470:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ba0. 00010000:00000001:0.0:1786176647.244444:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176647.244449:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176647.244455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176647.244458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176647.244462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176647.244464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176647.884429:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176647.884437:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176647.884440:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176647.884446:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176647.884450:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176647.884454:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176647.884457:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176647.884460:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176647.884463:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176648.268459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176648.268465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176648.268473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176648.268479:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176648.268484:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176648.268491:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176648.268496:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176648.268498:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176648.268500:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176648.268502:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176648.268506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176648.268508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176648.268561:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176648.268565:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176648.268568:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176648.397122:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176648.397128:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176648.397757:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176648.397759:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176648.398062:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176648.398065:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176648.398382:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176648.398384:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176648.398600:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176648.398602:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1786176648.908423:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176648.908431:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176648.908433:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176648.908438:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176648.908442:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176648.908445:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176648.908447:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176648.908449:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176648.908452:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176649.292449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176649.292453:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176649.292459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176649.292462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176649.292469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176649.292471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176649.932485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176649.932493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176649.932495:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176649.932500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176649.932504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176649.932507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176649.932510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176649.932512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176649.932515:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176650.316504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176650.316509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176650.316515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176650.316517:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176650.316525:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176650.316526:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176650.316592:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176650.316596:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176650.316600:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176651.340443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176651.340449:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176651.340452:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176651.340457:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176651.340463:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176651.340468:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176651.340470:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176651.340475:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176651.340479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176651.340481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176651.340484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176651.340486:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176651.340487:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176651.340488:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176651.340490:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176651.340491:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176651.340492:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176651.340493:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176652.300466:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4305999104, at 4305999168 00000100:00000010:0.0:1786176652.300485:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62c60. 00010000:00000001:0.0:1786176652.364486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176652.364494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176652.364509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176652.364511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176652.364515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176652.364516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176652.364638:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176652.364644:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176652.364649:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176653.388493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176653.388499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176653.388504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176653.388507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176653.388512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176653.388513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176653.517625:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176653.517630:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176653.518261:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176653.518264:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176653.518763:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176653.518766:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176653.518982:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176653.518985:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1786176653.519234:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176653.519236:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00010000:00000001:0.0:1786176654.412507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176654.412513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176654.412518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176654.412521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176654.412524:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176654.412528:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176654.412603:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176654.412607:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176654.412611:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176655.052503:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176655.052513:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176655.052515:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176655.052522:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176655.052526:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176655.052529:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176655.052532:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176655.052542:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176655.052545:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176655.436464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176655.436470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176655.436478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176655.436480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176655.436485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176655.436489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176656.076553:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176656.076578:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176656.076582:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176656.076589:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176656.076594:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176656.076599:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176656.076603:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176656.076607:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176656.076611:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176656.460465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176656.460470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176656.460477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176656.460479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176656.460482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176656.460484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176656.460581:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176656.460590:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176656.460612:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176657.100474:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176657.100485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176657.100487:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176657.100492:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176657.100499:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176657.100503:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176657.100506:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176657.100509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176657.100512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176657.420461:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306004225, at 4306004288 00000100:00000010:0.0:1786176657.420477:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62840. 00010000:00000001:0.0:1786176657.484449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176657.484458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176657.484464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176657.484469:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176657.484471:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176657.484476:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176657.484479:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176657.484482:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176657.484484:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176657.484486:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176657.484490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176657.484492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.124642:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176658.124649:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176658.124651:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176658.124655:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176658.124657:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176658.124660:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176658.124662:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176658.124664:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176658.124666:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176658.508511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176658.508522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176658.508553:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176658.508561:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176658.508573:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176658.508579:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176658.508641:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176658.508644:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176658.508646:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176658.636467:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176658.636481:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176658.636483:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176658.636487:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176658.636491:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176658.636499:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11333 00000400:00000200:3.0:1786176658.636506:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176658.636512:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176658.636514:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176658.636516:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636519:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176658.636521:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636523:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176658.636526:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176658.636528:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176658.636530:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176658.636533:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176658.636544:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176658.636548:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176658.636552:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636554:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176658.636556:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636558:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176658.636561:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176658.636565:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636567:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176658.636575:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176658.636577:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176658.636580:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.636581:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176658.636583:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176658.636591:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176658.636594:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176658.636597:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176658.636598:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.636601:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176658.636604:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176658.636607:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176658.636612:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176658.636617:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176658.636625:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235520000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176658.636633:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176658.636636:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176658.636639:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235520000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176658.636656:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176658.636658:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176658.636661:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176658.636662:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176658.636664:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176658.636666:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176658.636667:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11333 00000400:00000200:3.0:1786176658.636670:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176658.636672:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176658.636674:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176658.636676:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636678:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176658.636680:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636682:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176658.636683:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176658.636685:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176658.636687:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176658.636689:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176658.636690:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176658.636691:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176658.636693:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636694:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176658.636695:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636698:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176658.636700:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176658.636704:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636706:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176658.636707:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176658.636712:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176658.636713:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.636715:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176658.636718:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176658.636721:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.636723:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176658.636729:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176658.636733:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176658.636737:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176658.636743:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.636746:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176658.636748:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176658.636753:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176658.636758:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176658.636766:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176658.636783:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235520128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176658.636801:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176658.636811:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176658.636820:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235520128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176658.636851:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176658.636854:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176658.636861:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176658.636864:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176658.636871:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176658.636873:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176658.636887:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11333 00000400:00000200:3.0:1786176658.636893:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176658.636903:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176658.636911:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176658.636917:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636926:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176658.636930:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636932:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176658.636939:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176658.636944:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176658.636952:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176658.636954:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176658.636960:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176658.636970:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176658.636986:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.636997:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176658.637005:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637009:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176658.637012:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176658.637018:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637027:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176658.637031:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176658.637034:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176658.637037:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.637044:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176658.637047:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176658.637050:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.637051:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176658.637056:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176658.637061:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176658.637064:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176658.637075:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.637083:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176658.637086:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176658.637092:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176658.637098:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176658.637109:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176658.637126:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235520256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176658.637142:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176658.637149:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176658.637159:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235520256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176658.637184:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176658.637194:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176658.637198:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176658.637199:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176658.637201:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176658.637205:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176658.637210:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11333 00000400:00000200:3.0:1786176658.637219:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176658.637227:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176658.637232:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1786176658.637237:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786176658.637238:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637254:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1786176658.637255:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00000040:3.0:1786176658.637263:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637272:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176658.637283:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176658.637289:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176658.637292:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176658.637295:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176658.637306:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176658.637318:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176658.637332:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637336:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176658.637344:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637349:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176658.637386:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176658.637389:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637391:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176658.637393:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176658.637396:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176658.637400:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.637402:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176658.637405:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176658.637408:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.637410:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176658.637418:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176658.637427:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176658.637434:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176658.637440:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.637451:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176658.637457:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176658.637460:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176658.637472:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176658.637483:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176658.637492:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235520384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176658.637500:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176658.637504:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176658.637508:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235520384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176658.637522:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176658.637528:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176658.637531:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176658.637533:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176658.637546:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176658.637555:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176658.637559:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11333 00000400:00000200:3.0:1786176658.637565:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176658.637571:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176658.637573:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176658.637577:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637579:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176658.637581:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637585:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176658.637587:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176658.637588:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176658.637588:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176658.637592:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176658.637595:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176658.637609:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176658.637615:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637625:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176658.637631:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637638:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176658.637641:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176658.637650:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176658.637660:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176658.637667:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176658.637674:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176658.637681:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.637688:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176658.637699:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176658.637707:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.637720:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176658.637729:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176658.637735:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176658.637747:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176658.637751:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176658.637757:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176658.637763:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176658.637769:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176658.637780:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176658.637790:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176658.637804:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235520512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176658.637865:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176658.637874:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176658.637889:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235520512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176658.637926:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176658.637938:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11343) 00000800:00000200:1.1:1786176658.638103:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176658.638105:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176658.638299:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176658.638301:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176658.638565:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176658.638569:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176658.638765:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176658.638767:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1786176659.148431:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176659.148440:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176659.148442:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176659.148447:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176659.148450:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176659.148453:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176659.148466:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176659.148468:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176659.148470:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176659.532429:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176659.532435:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176659.532440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176659.532441:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176659.532445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176659.532446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1786176660.108549:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:3.0:1786176660.108585:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:3.0:1786176660.108590:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1786176660.108755:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1786176660.108778:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176660.108790:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786176660.108796:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176660.108798:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176660.172478:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176660.172484:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176660.172486:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176660.172501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176660.172505:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176660.172508:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176660.172510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176660.172512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176660.172515:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176660.556472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176660.556477:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176660.556481:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176660.556485:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176660.556490:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176660.556493:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176660.556496:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176660.556498:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176660.556501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176660.556502:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176660.556505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176660.556506:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176660.556507:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176660.556508:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176660.556509:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176660.556510:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176660.556511:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176660.556513:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176660.556576:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176660.556580:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176660.556583:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176661.196506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176661.196526:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176661.196529:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176661.196535:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176661.196540:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176661.196545:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176661.196549:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176661.196552:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176661.196556:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176661.580461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176661.580465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176661.580471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176661.580473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176661.580476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176661.580479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176662.220469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176662.220478:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176662.220481:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176662.220487:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176662.220492:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176662.220496:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176662.220499:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176662.220515:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176662.220519:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176662.540471:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306009345, at 4306009408 00000100:00000010:0.0:1786176662.540486:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62540. 00010000:00000001:0.0:1786176662.604476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176662.604482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176662.604489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176662.604491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176662.604494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176662.604511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176662.604559:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176662.604562:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176662.604565:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176663.244506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176663.244514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176663.244517:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176663.244523:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176663.244528:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176663.244532:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176663.244535:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176663.244538:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176663.244542:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176663.628452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176663.628455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176663.628461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176663.628462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176663.628466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176663.628467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176663.756464:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176663.756472:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176663.756474:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176663.756477:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176663.756480:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176663.756498:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11338 00000400:00000200:3.0:1786176663.756506:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176663.756511:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176663.756514:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176663.756516:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756519:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176663.756521:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756522:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176663.756526:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176663.756527:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176663.756529:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176663.756532:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176663.756535:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176663.756538:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176663.756541:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756543:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176663.756545:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756547:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176663.756551:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176663.756554:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756557:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176663.756560:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176663.756563:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176663.756566:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.756569:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176663.756572:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176663.756582:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786176663.756586:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176663.756589:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176663.756591:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.756594:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176663.756596:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176663.756600:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176663.756606:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176663.756611:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176663.756619:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235520640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176663.756626:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176663.756630:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176663.756633:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235520640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176663.756651:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176663.756653:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176663.756656:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176663.756658:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176663.756659:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176663.756661:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176663.756662:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11338 00000400:00000200:3.0:1786176663.756666:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176663.756668:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176663.756669:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176663.756671:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756673:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176663.756675:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756677:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176663.756680:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176663.756682:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176663.756683:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176663.756685:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176663.756686:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176663.756688:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176663.756691:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756694:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176663.756696:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756699:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176663.756703:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176663.756708:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756710:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176663.756714:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176663.756720:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176663.756730:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.756733:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176663.756739:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176663.756744:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.756748:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176663.756752:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786176663.756755:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176663.756759:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176663.756761:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.756763:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176663.756764:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176663.756769:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176663.756777:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176663.756781:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176663.756788:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235520768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176663.756798:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176663.756801:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176663.756808:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235520768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176663.756821:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176663.756826:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176663.756831:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176663.756836:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176663.756839:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176663.756844:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176663.756849:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11338 00000400:00000200:3.0:1786176663.756855:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176663.756860:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176663.756862:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176663.756867:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756871:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176663.756873:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756875:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176663.756877:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176663.756881:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176663.756884:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176663.756887:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176663.756890:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176663.756893:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176663.756899:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756904:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176663.756910:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756914:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176663.756918:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176663.756923:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176663.756928:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176663.756932:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176663.756936:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176663.756943:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.756947:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176663.756950:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176663.756954:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.756955:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176663.756959:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786176663.756962:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176663.756964:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176663.756966:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.756968:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176663.756969:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176663.756971:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176663.756975:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176663.756979:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176663.756985:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235520896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176663.756996:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176663.756999:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176663.757003:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235520896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176663.757017:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176663.757021:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1786176663.757024:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786176663.757026:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176663.757029:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.1:1786176663.757029:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00080000:3.0:1786176663.757032:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176663.757037:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176663.757043:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11338 00000400:00000200:3.0:1786176663.757048:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176663.757053:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176663.757057:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176663.757063:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.757066:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176663.757071:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.757075:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176663.757079:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176663.757080:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176663.757082:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176663.757084:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176663.757089:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176663.757094:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176663.757101:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.757104:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176663.757109:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.757114:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176663.757120:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176663.757125:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176663.757129:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176663.757131:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176663.757134:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176663.757140:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.757141:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176663.757144:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176663.757148:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.757152:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176663.757156:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176663.757160:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176663.757162:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176663.757165:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.757167:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176663.757169:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176663.757171:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176663.757175:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176663.757180:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176663.757189:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235521024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176663.757201:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176663.757205:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176663.757213:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235521024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176663.757239:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176663.757242:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176663.757248:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176663.757252:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176663.757254:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176663.757258:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176663.757263:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11338 00000400:00000200:3.0:1786176663.757268:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176663.757274:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176663.757278:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176663.757284:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.757290:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176663.757294:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.757296:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176663.757304:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176663.757308:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176663.757312:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176663.757317:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176663.757324:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176663.757330:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176663.757342:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.757349:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176663.757354:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176663.757385:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176663.757389:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176663.757395:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176663.757399:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176663.757403:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176663.757407:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176663.757412:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.757414:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176663.757418:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176663.757422:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.757426:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176663.757431:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176663.757436:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176663.757440:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176663.757442:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176663.757446:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176663.757449:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176663.757451:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176663.757456:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176663.757461:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176663.757470:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235521152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176663.757483:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176663.757499:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176663.757503:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235521152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176663.757518:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176663.757524:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11348) 00000800:00000200:1.1:1786176663.757619:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176663.757621:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176663.757805:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176663.757806:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176663.758007:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176663.758008:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176663.758264:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176663.758266:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:0.0:1786176664.268467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176664.268475:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176664.268477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176664.268508:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176664.268512:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176664.268521:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176664.268523:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176664.268526:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176664.268528:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176664.652457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176664.652462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176664.652466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176664.652469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176664.652471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176664.652472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176664.652549:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176664.652554:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176664.652557:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176665.292455:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176665.292462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176665.292464:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176665.292468:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176665.292495:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176665.292498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176665.292501:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176665.292503:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176665.292506:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176665.676488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176665.676493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176665.676503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176665.676507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176665.676511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176665.676512:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176666.316499:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176666.316508:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176666.316511:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176666.316519:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176666.316524:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176666.316528:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176666.316531:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176666.316535:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176666.316538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176666.700492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176666.700504:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176666.700512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176666.700515:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176666.700522:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176666.700525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176667.340488:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176667.340498:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176667.340500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176667.340506:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176667.340511:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176667.340515:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176667.340518:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176667.340531:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176667.340535:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176667.660441:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306014464, at 4306014528 00000100:00000010:0.0:1786176667.660471:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c626c0. 00010000:00000001:0.0:1786176667.724490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176667.724496:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176667.724503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176667.724508:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176667.724514:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176667.724519:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176667.724526:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176667.724530:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176667.724532:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176667.724535:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176667.724541:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176667.724543:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176668.364480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176668.364488:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176668.364491:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176668.364497:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176668.364502:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176668.364507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176668.364511:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176668.364528:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176668.364534:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176668.748434:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176668.748450:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176668.748455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176668.748457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176668.748462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176668.748464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176668.877054:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176668.877063:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176668.877598:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176668.877600:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176668.877840:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176668.877842:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176668.878098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176668.878099:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176668.878285:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176668.878286:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1786176669.388459:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176669.388467:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176669.388470:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176669.388476:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176669.388489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176669.388493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176669.388496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176669.388499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176669.388502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176669.772500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176669.772505:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176669.772513:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176669.772516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176669.772522:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176669.772525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176670.412488:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176670.412496:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176670.412499:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176670.412506:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176670.412510:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176670.412514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176670.412517:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176670.412521:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176670.412525:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176670.796463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176670.796473:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176670.796476:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176670.796480:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176670.796488:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176670.796492:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176670.796496:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176670.796499:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176670.796508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176670.796512:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176670.796518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176670.796522:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176670.796526:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176670.796530:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176670.796532:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176670.796535:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176670.796537:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176670.796540:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176671.436465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176671.436479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176671.436482:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176671.436489:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176671.436494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176671.436499:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176671.436502:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176671.436506:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176671.436510:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176671.820500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176671.820503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176671.820507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176671.820508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176671.820512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176671.820513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176672.460474:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176672.460483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176672.460486:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176672.460493:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176672.460498:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176672.460501:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176672.460504:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176672.460507:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176672.460511:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176672.780489:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306019584, at 4306019648 00000100:00000010:0.0:1786176672.780504:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cae0. 00010000:00000001:0.0:1786176672.844496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176672.844502:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176672.844507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176672.844510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176672.844515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176672.844517:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176673.484456:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176673.484473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176673.484477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176673.484484:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176673.484489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176673.484494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176673.484497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176673.484500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176673.484503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176673.868512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176673.868516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176673.868521:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176673.868523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176673.868526:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176673.868528:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176673.997082:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176673.997091:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176673.997866:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176673.997873:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176673.998227:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176673.998229:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176673.998535:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176673.998538:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176673.998739:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176673.998741:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1786176674.508486:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176674.508494:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176674.508496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176674.508503:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176674.508506:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176674.508510:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176674.508512:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176674.508515:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176674.508518:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176674.892479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176674.892486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176674.892495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176674.892499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176674.892507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176674.892510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176675.532488:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176675.532495:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176675.532498:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176675.532503:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176675.532507:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176675.532510:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176675.532513:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176675.532515:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176675.532519:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176675.916460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176675.916465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176675.916470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176675.916473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176675.916478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176675.916482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176675.916549:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176675.916554:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176675.916558:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176676.556447:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176676.556456:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176676.556459:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176676.556464:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176676.556468:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176676.556472:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176676.556475:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176676.556478:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176676.556481:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176676.940459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176676.940465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176676.940473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176676.940478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176676.940486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176676.940491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176677.580416:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176677.580428:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176677.580431:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176677.580438:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176677.580443:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176677.580448:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176677.580451:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176677.580455:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176677.580459:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176677.900434:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306024704, at 4306024768 00000100:00000010:0.0:1786176677.900450:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c720. 00010000:00000001:0.0:1786176677.964456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176677.964464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176677.964473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176677.964478:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176677.964483:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176677.964488:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176677.964495:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176677.964499:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176677.964516:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176677.964524:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176677.964529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176677.964530:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176677.964581:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176677.964585:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176677.964589:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176678.604408:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176678.604414:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176678.604416:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176678.604421:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176678.604425:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176678.604429:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176678.604431:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176678.604434:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176678.604437:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176678.988462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176678.988471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176678.988481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176678.988490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176678.988497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176678.988500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176679.117456:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176679.117461:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176679.118827:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176679.118831:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176679.119180:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176679.119183:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176679.119416:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176679.119417:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786176679.119686:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176679.119688:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1786176679.628432:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176679.628444:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176679.628448:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176679.628454:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176679.628458:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176679.628462:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176679.628464:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176679.628466:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176679.628469:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176680.012449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176680.012457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176680.012464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176680.012465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176680.012468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176680.012470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176680.012524:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176680.012529:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176680.012534:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176680.652416:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176680.652423:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176680.652425:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176680.652430:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176680.652433:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176680.652437:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176680.652439:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176680.652441:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176680.652444:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176681.036443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176681.036449:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176681.036452:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176681.036456:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176681.036459:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176681.036460:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176681.036462:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176681.036464:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176681.036466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176681.036467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176681.036470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176681.036471:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176681.036472:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176681.036473:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176681.036474:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176681.036475:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176681.036476:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176681.036477:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176681.676471:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176681.676483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176681.676487:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176681.676495:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176681.676501:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176681.676506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176681.676510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176681.676513:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176681.676517:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176682.060459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176682.060466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176682.060473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176682.060479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176682.060485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176682.060495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176682.060583:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176682.060587:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176682.060590:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176682.700433:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176682.700442:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176682.700445:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176682.700452:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176682.700456:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176682.700460:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176682.700464:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176682.700467:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176682.700470:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176683.020456:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306029825, at 4306029888 00000100:00000010:0.0:1786176683.020480:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cd80. 00010000:00000001:0.0:1786176683.084489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176683.084499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176683.084510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176683.084514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176683.084521:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176683.084523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176683.724490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176683.724498:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176683.724499:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176683.724504:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176683.724508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176683.724512:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176683.724514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176683.724517:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176683.724520:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176684.108460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176684.108465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176684.108472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176684.108475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176684.108480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176684.108484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176684.108545:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176684.108549:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176684.108551:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176684.237039:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176684.237043:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176684.237666:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176684.237668:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176684.237906:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176684.237908:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176684.238089:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176684.238091:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176684.238434:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176684.238436:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1786176684.748441:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176684.748448:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176684.748450:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176684.748462:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176684.748466:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176684.748469:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176684.748471:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176684.748473:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176684.748476:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176685.132467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176685.132471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176685.132478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176685.132481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176685.132487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176685.132490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176685.772483:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176685.772490:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176685.772492:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176685.772496:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176685.772500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176685.772503:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176685.772505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176685.772507:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176685.772510:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176686.156457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176686.156461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176686.156467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176686.156468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176686.156472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176686.156473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176686.156520:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176686.156523:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176686.156526:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176686.796548:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176686.796566:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176686.796569:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176686.796578:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176686.796585:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176686.796591:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176686.796594:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176686.796599:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176686.796603:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176687.180484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176687.180490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176687.180499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176687.180504:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176687.180508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176687.180512:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176687.180579:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176687.180583:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176687.180586:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176687.820574:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176687.820594:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176687.820598:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176687.820609:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176687.820616:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176687.820623:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176687.820633:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176687.820639:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176687.820645:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176688.140433:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306034944, at 4306035008 00000100:00000010:0.0:1786176688.140459:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c625a0. 00010000:00000001:0.0:1786176688.204465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176688.204470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176688.204478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176688.204482:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176688.204485:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176688.204491:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176688.204499:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176688.204504:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176688.204508:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176688.204511:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176688.204517:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176688.204519:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176688.204583:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176688.204588:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176688.204592:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176688.844609:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176688.844623:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176688.844626:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176688.844633:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176688.844638:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176688.844644:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176688.844647:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176688.844654:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176688.844658:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176689.228509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176689.228516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176689.228524:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176689.228530:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176689.228536:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176689.228539:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176689.228614:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176689.228618:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176689.228622:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176689.357004:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176689.357009:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176689.357670:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176689.357671:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176689.357892:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176689.357893:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176689.358074:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176689.358076:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176689.358338:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176689.358340:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00000001:3.0:1786176689.869162:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176689.869172:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176689.869175:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176689.869181:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176689.869187:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176689.869192:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176689.869195:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176689.869199:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176689.869202:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176690.252480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176690.252485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176690.252495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176690.252498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176690.252503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176690.252506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176690.252574:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176690.252581:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176690.252587:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176691.276501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176691.276510:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176691.276513:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176691.276517:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176691.276522:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176691.276526:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176691.276532:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176691.276539:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176691.276557:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176691.276565:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176691.276580:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176691.276586:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176691.276595:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176691.276601:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176691.276604:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176691.276605:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176691.276607:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176691.276610:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176691.276667:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176691.276671:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176691.276675:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176691.916845:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176691.916854:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176691.916857:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176691.916862:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176691.916866:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176691.916918:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176691.916921:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176691.916923:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176691.916925:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176692.300440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176692.300446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176692.300451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176692.300452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176692.300456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176692.300457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176692.300516:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176692.300523:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176692.300527:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176692.940461:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176692.940471:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176692.940475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176692.940482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176692.940487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176692.940492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176692.940496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176692.940500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176692.940504:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176693.260526:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306040064, at 4306040128 00000100:00000010:0.0:1786176693.260551:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62de0. 00010000:00000001:0.0:1786176693.324511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176693.324519:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176693.324535:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176693.324540:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176693.324549:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176693.324553:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176693.324613:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176693.324617:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176693.324620:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176693.964496:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176693.964504:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176693.964506:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176693.964511:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176693.964526:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176693.964530:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176693.964532:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176693.964535:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176693.964538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176694.348481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176694.348488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176694.348496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176694.348500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176694.348509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176694.348514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176694.348586:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176694.348591:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176694.348595:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176694.476489:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176694.476497:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176694.476500:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176694.476503:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176694.476507:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176694.476514:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11368 00000400:00000200:3.0:1786176694.476521:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176694.476527:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176694.476529:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176694.476532:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476534:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176694.476547:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476549:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176694.476552:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176694.476554:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176694.476556:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176694.476558:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176694.476560:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176694.476562:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176694.476564:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476565:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176694.476566:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476568:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176694.476571:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176694.476573:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476575:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176694.476577:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176694.476580:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176694.476582:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.476583:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176694.476585:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176694.476594:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176694.476596:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176694.476600:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176694.476601:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.476603:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176694.476606:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176694.476609:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176694.476616:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176694.476622:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176694.476629:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235524480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176694.476637:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176694.476641:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176694.476644:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235524480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176694.476662:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176694.476665:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176694.476674:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176694.476675:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176694.476677:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176694.476679:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176694.476680:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11368 00000400:00000200:3.0:1786176694.476683:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176694.476685:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176694.476688:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176694.476690:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476692:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176694.476694:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476696:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176694.476697:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176694.476698:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176694.476700:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176694.476701:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176694.476702:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176694.476704:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176694.476708:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476711:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176694.476713:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476715:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176694.476716:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176694.476718:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476720:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176694.476721:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176694.476725:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176694.476727:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.476729:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176694.476732:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176694.476739:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.476740:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176694.476744:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176694.476747:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176694.476751:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176694.476754:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.476755:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176694.476756:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176694.476757:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176694.476763:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176694.476770:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176694.476788:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235524608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176694.476819:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176694.476827:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176694.476843:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235524608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176694.476876:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176694.476879:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176694.476883:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176694.476885:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176694.476886:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176694.476889:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176694.476901:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11368 00000400:00000200:3.0:1786176694.476906:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176694.476916:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176694.476920:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176694.476926:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476930:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176694.476938:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476940:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176694.476944:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176694.476951:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176694.476957:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176694.476967:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176694.476970:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176694.476973:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176694.476981:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.476992:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176694.476997:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477002:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176694.477014:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176694.477021:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477025:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176694.477033:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176694.477037:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176694.477051:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.477054:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176694.477056:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176694.477066:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.477083:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176694.477089:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176694.477097:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176694.477100:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176694.477104:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.477108:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176694.477114:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176694.477123:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176694.477132:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176694.477141:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176694.477154:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235524736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176694.477166:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176694.477175:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176694.477233:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235524736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176694.477262:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176694.477266:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176694.477274:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176694.477277:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176694.477281:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176694.477288:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176694.477298:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11368 00000800:00000200:1.1:1786176694.477307:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:3.0:1786176694.477308:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:1.1:1786176694.477315:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000400:00000200:3.0:1786176694.477332:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176694.477339:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176694.477350:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477389:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176694.477395:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477399:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176694.477403:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176694.477405:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176694.477408:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176694.477410:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176694.477412:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176694.477414:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176694.477418:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477421:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176694.477424:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477427:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176694.477430:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176694.477433:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477437:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176694.477447:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176694.477464:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176694.477469:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.477476:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176694.477482:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176694.477487:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.477491:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176694.477496:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176694.477500:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176694.477509:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176694.477516:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.477526:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176694.477533:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176694.477537:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176694.477550:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176694.477556:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176694.477564:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235524864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176694.477579:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176694.477583:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176694.477601:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235524864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176694.477623:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176694.477627:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176694.477633:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176694.477637:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176694.477639:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176694.477642:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176694.477647:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11368 00000400:00000200:3.0:1786176694.477660:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176694.477667:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176694.477669:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176694.477672:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477677:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176694.477681:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477688:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176694.477697:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176694.477702:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176694.477707:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176694.477716:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176694.477719:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176694.477720:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176694.477740:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477743:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176694.477748:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477756:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176694.477766:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176694.477777:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176694.477783:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176694.477799:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176694.477806:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176694.477814:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.477827:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176694.477830:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176694.477835:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.477844:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176694.477854:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176694.477860:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176694.477865:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176694.477870:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176694.477877:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176694.477880:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176694.477885:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176694.477897:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176694.477905:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176694.477913:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235524992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176694.477925:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176694.477929:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176694.477940:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235524992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176694.477957:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176694.477962:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11379) 00000800:00000200:1.1:1786176694.478181:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176694.478183:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176694.478486:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176694.478488:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176694.478785:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176694.478788:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1786176694.479067:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176694.479069:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000100:00000001:3.0:1786176694.988501:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176694.988509:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176694.988513:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176694.988519:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176694.988524:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176694.988529:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176694.988532:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176694.988535:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176694.988538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176695.372496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176695.372514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176695.372530:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176695.372537:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176695.372543:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176695.372545:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176695.372602:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176695.372607:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176695.372610:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176696.012486:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176696.012495:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176696.012497:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176696.012503:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176696.012508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176696.012514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176696.012518:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176696.012522:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176696.012526:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176696.396453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176696.396460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176696.396464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176696.396465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176696.396470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176696.396472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176696.396539:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176696.396543:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176696.396546:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176697.036472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176697.036481:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176697.036483:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176697.036490:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176697.036495:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176697.036499:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176697.036504:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176697.036508:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176697.036512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176697.420484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176697.420488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176697.420496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176697.420500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176697.420504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176697.420507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176697.420580:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176697.420585:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176697.420589:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786176697.996458:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786176697.996466:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786176697.996470:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1786176697.996639:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176697.996657:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786176697.996663:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176697.996666:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176698.060444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176698.060452:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176698.060455:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176698.060461:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176698.060465:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176698.060468:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176698.060471:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176698.060474:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176698.060477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176698.380477:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306045185, at 4306045248 00000100:00000010:0.0:1786176698.380494:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c629c0. 00010000:00000001:0.0:1786176698.444440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176698.444447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176698.444453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176698.444456:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176698.444460:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176698.444464:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176698.444468:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176698.444470:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176698.444474:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176698.444476:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176698.444483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176698.444486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176698.444528:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176698.444532:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176698.444534:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176699.084464:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176699.084473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176699.084476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176699.084482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176699.084487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176699.084491:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176699.084495:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176699.084498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176699.084501:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176699.468446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176699.468450:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176699.468455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176699.468458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176699.468461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176699.468463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176699.468525:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176699.468527:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176699.468530:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176699.597005:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176699.597011:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786176699.597666:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176699.597668:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786176699.597857:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176699.597858:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786176699.598071:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176699.598072:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786176699.598387:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176699.598389:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000100:00000001:3.0:1786176700.108440:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176700.108447:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176700.108449:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176700.108453:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176700.108456:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176700.108459:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176700.108461:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176700.108463:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176700.108465:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176700.492449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176700.492456:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176700.492459:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176700.492462:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176700.492465:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176700.492466:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176700.492468:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176700.492470:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176700.492473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176700.492474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176700.492476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176700.492477:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176700.492479:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176700.492480:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176700.492481:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176700.492482:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176700.492484:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176700.492485:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176700.492525:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176700.492529:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176700.492543:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176701.132491:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176701.132501:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176701.132503:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176701.132509:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176701.132513:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176701.132517:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176701.132519:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176701.132522:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176701.132526:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176701.516453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176701.516456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176701.516463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176701.516464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176701.516468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176701.516469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176701.516533:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176701.516537:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176701.516540:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176702.156455:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176702.156464:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176702.156467:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176702.156473:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176702.156477:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176702.156481:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176702.156483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176702.156486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176702.156490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176702.540477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176702.540481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176702.540487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176702.540489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176702.540492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176702.540494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176702.540564:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176702.540570:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176702.540573:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176703.180486:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176703.180495:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176703.180498:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176703.180504:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176703.180508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176703.180511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176703.180514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176703.180517:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176703.180520:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176703.500471:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306050304, at 4306050368 00000100:00000010:0.0:1786176703.500485:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c625a0. 00010000:00000001:0.0:1786176703.564462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176703.564467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176703.564472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176703.564474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176703.564479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176703.564481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176703.564554:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176703.564558:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176703.564560:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176704.588458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176704.588462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176704.588471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176704.588474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176704.588479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176704.588482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176704.588555:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176704.588560:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176704.588564:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176704.716984:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176704.717417:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176704.717720:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176704.717864:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176704.718031:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786176705.228515:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176705.228524:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176705.228527:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176705.228533:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176705.228538:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176705.228542:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176705.228545:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176705.228547:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176705.228552:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176705.612449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176705.612454:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176705.612459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176705.612462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176705.612468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176705.612471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176705.612543:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176705.612547:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176705.612551:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176706.252483:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176706.252490:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176706.252492:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176706.252496:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176706.252507:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176706.252510:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176706.252513:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176706.252515:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176706.252518:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176706.636433:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176706.636438:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176706.636444:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176706.636447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176706.636453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176706.636458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176706.636530:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176706.636534:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176706.636537:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176707.276457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176707.276465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176707.276467:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176707.276471:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176707.276475:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176707.276478:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176707.276480:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176707.276483:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176707.276488:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176707.660410:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176707.660432:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176707.660438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176707.660439:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176707.660442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176707.660443:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176707.660501:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176707.660507:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176707.660513:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1786176707.788630:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:3.0:1786176707.788667:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1786176707.788677:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176707.788680:0:2381:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:3.0:1786176707.788684:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176707.788687:0:2381:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1786176707.788691:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176707.788694:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:3.0:1786176707.788698:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176707.788702:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176707.788706:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28990 00000400:00000010:3.0:1786176707.788708:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28990. 00000100:00000001:3.0:1786176707.788712:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176707.788719:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328b9f80 x1872930235525760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176707.788746:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176707.788750:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235525760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176729 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176707.788765:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176707.788767:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176707.788779:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176707.788782:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d900. 00000400:00000200:3.0:1786176707.788786:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176707.788791:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176707.788795:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28198 00000400:00000010:3.0:1786176707.788797:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28198. 00000100:00000001:3.0:1786176707.788800:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176707.788806:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328b8a80 x1872930235525888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176707.788819:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176707.788828:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8a80 x1872930235525888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176729 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176707.788861:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176707.788864:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176707.788872:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176707.788875:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6db00. 00000400:00000200:3.0:1786176707.788891:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176707.788899:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176707.788906:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c286e8 00000400:00000010:3.0:1786176707.788910:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c286e8. 00000100:00000001:3.0:1786176707.788916:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176707.788929:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328bb800 x1872930235526016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176707.788944:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176707.788953:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb800 x1872930235526016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176729 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176707.788969:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176707.788974:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176707.788985:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176707.788991:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d100. 00000400:00000200:3.0:1786176707.788999:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176707.789008:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176707.789016:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28220 00000400:00000010:3.0:1786176707.789021:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28220. 00000100:00000001:3.0:1786176707.789027:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176707.789037:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328b9180 x1872930235526144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176707.789095:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176707.789104:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9180 x1872930235526144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176729 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176707.789126:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176707.789131:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176707.789142:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176707.789148:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6df00. 00000400:00000200:3.0:1786176707.789154:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176707.789164:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176707.789171:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28e58 00000400:00000010:3.0:1786176707.789176:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28e58. 00000100:00000001:3.0:1786176707.789182:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176707.789190:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328bb100 x1872930235526272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176707.789203:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176707.789211:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb100 x1872930235526272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176729 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176707.789223:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176707.789227:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786176708.300460:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176708.300469:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176708.300471:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176708.300478:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176708.300481:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176708.300485:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176708.300489:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176708.300492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176708.300495:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176708.620440:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306055424, at 4306055488 00000100:00000010:0.0:1786176708.620458:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c060. 00010000:00000001:0.0:1786176708.684493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176708.684499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176708.684512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176708.684517:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176708.684522:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176708.684532:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176708.684542:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176708.684545:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176708.684548:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176708.684552:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176708.684558:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176708.684559:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176708.684620:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176708.684630:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176708.684633:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176709.324471:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176709.324481:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176709.324483:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176709.324489:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176709.324495:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176709.324499:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176709.324503:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176709.324507:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176709.324511:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176709.708435:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176709.708441:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176709.708446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176709.708447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176709.708449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176709.708450:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176709.708515:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176709.708521:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176709.708525:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176709.837698:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176709.837709:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176709.838536:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176709.838539:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176709.838877:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176709.838879:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176709.839213:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176709.839215:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176709.839466:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176709.839468:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1786176710.348459:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176710.348471:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176710.348475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176710.348482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176710.348487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176710.348494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176710.348497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176710.348501:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176710.348505:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176710.732503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176710.732512:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176710.732516:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176710.732523:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176710.732534:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176710.732539:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176710.732544:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176710.732549:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176710.732554:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176710.732556:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176710.732561:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176710.732563:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176710.732565:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176710.732568:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176710.732570:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176710.732572:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176710.732574:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176710.732576:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176710.732643:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176710.732648:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176710.732651:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176711.372480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176711.372491:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176711.372494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176711.372501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176711.372509:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176711.372514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176711.372517:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176711.372522:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176711.372529:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176711.756510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176711.756518:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176711.756528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176711.756532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176711.756538:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176711.756543:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176711.756629:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176711.756632:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176711.756635:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176712.780495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176712.780502:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176712.780510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176712.780516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176712.780521:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176712.780526:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176712.780580:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176712.780584:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176712.780587:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176713.420539:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176713.420546:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176713.420549:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176713.420553:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176713.420557:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176713.420560:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176713.420562:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176713.420565:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176713.420569:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176713.740507:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306060545, at 4306060608 00000100:00000010:0.0:1786176713.740527:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cc60. 00010000:00000001:0.0:1786176713.804506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176713.804513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176713.804525:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176713.804527:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176713.804532:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176713.804538:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176713.804613:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176713.804619:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176713.804623:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176714.444506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176714.444515:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176714.444517:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176714.444523:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176714.444527:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176714.444531:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176714.444533:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176714.444535:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176714.444538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176714.828497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176714.828508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176714.828519:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176714.828526:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176714.828536:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176714.828540:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176714.828616:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176714.828620:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176714.828622:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176714.956501:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176714.956512:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176714.956515:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176714.956519:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176714.956524:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176714.956532:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11389 00000400:00000200:3.0:1786176714.956541:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176714.956548:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176714.956552:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176714.956555:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.956559:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176714.956563:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.956565:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176714.956570:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176714.956572:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176714.956575:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176714.956579:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176714.956581:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176714.956584:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176714.956588:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.956590:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176714.956592:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.956594:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176714.956598:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176714.956601:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176714.956604:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176714.956607:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176714.956611:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176714.956615:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.956617:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176714.956619:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176714.956630:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176714.956634:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176714.956639:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176714.956641:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.956644:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176714.956647:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176714.956651:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176714.956660:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176714.956666:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176714.956677:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235527040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176714.956689:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176714.956694:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176714.956699:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235527040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176714.956727:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176714.956731:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176714.956734:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176714.956736:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176714.956738:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176714.956741:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176714.956743:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11389 00000400:00000200:3.0:1786176714.956748:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176714.956752:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176714.956756:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176714.956759:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.956762:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176714.956765:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.956769:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176714.956777:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176714.956779:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176714.956787:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176714.956791:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176714.956796:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176714.956800:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176714.956808:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.956816:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176714.956822:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.956832:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176714.956842:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176714.956853:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176714.956865:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176714.956869:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176714.956876:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176714.956883:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.956892:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176714.956900:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176714.956906:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.956911:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176714.956919:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176714.956925:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176714.956930:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176714.956931:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.956936:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176714.956943:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176714.956949:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176714.956958:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176714.956968:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176714.956985:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235527168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176714.957014:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176714.957019:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176714.957029:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235527168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176714.957098:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176714.957106:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176714.957113:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176714.957116:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176714.957119:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176714.957133:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176714.957140:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11389 00000400:00000200:3.0:1786176714.957191:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176714.957210:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176714.957217:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176714.957226:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.957231:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176714.957238:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.957245:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176714.957253:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176714.957256:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176714.957265:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:2.1:1786176714.957270:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786176714.957271:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000800:00000200:2.1:1786176714.957279:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000020:00000001:3.0:1786176714.957280:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176714.957291:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176714.957304:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.957312:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176714.957320:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.957332:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176714.957409:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176714.957420:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176714.957435:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176714.957441:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176714.957446:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176714.957453:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.957457:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176714.957462:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176714.957472:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.957477:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176714.957486:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176714.957492:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176714.957501:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176714.957506:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.957513:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176714.957516:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176714.957524:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176714.957531:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176714.957539:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176714.957549:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235527296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176714.957566:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176714.957572:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176714.957583:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235527296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176714.957607:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176714.957615:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176714.957630:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176714.957632:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176714.957634:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176714.957638:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176714.957644:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11389 00000400:00000200:3.0:1786176714.957654:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176714.957668:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176714.957679:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176714.957692:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.957699:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176714.957704:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.957714:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176714.957721:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176714.957726:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176714.957731:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176714.957736:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176714.957743:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176714.957749:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176714.957760:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.957766:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176714.957774:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.957785:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176714.957797:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176714.957811:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176714.957824:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176714.957831:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176714.957838:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176714.957844:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.957852:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176714.957858:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176714.957867:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.957873:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176714.957883:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176714.957891:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176714.957901:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176714.957908:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.957915:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176714.957918:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176714.957923:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176714.957932:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176714.957941:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176714.957954:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235527424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176714.958006:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176714.958011:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176714.958026:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235527424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176714.958044:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176714.958050:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176714.958061:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176714.958065:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176714.958071:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176714.958078:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176714.958086:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11389 00000400:00000200:3.0:1786176714.958099:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176714.958104:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176714.958109:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176714.958116:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.958122:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176714.958129:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.958137:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176714.958143:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176714.958147:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176714.958152:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176714.958158:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176714.958166:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176714.958173:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176714.958182:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.958188:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176714.958193:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176714.958198:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176714.958204:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176714.958217:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176714.958223:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176714.958227:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176714.958233:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176714.958239:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.958246:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176714.958250:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176714.958257:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.958264:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176714.958272:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176714.958279:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176714.958286:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176714.958292:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176714.958297:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176714.958301:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176714.958305:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176714.958313:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176714.958323:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176714.958340:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235527552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176714.958391:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176714.958395:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176714.958407:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235527552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176714.958440:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176714.958449:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11399) 00000800:00000200:2.1:1786176714.958501:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176714.958505:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176714.958819:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176714.958822:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176714.959125:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176714.959127:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786176714.959411:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176714.959413:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1786176715.468490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176715.468501:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176715.468503:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176715.468511:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176715.468515:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176715.468519:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176715.468522:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176715.468525:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176715.468528:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176715.852495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176715.852501:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176715.852506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176715.852507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176715.852510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176715.852511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176715.852580:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176715.852586:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176715.852590:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176716.492467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176716.492477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176716.492480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176716.492486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176716.492490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176716.492494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176716.492497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176716.492500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176716.492503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176716.876469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176716.876475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176716.876485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176716.876489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176716.876496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176716.876499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176716.876565:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176716.876570:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176716.876574:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176717.516448:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176717.516457:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176717.516460:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176717.516466:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176717.516471:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176717.516475:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176717.516478:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176717.516481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176717.516484:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176717.900443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176717.900447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176717.900453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176717.900455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176717.900458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176717.900462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176717.900529:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176717.900534:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176717.900538:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176718.540458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176718.540467:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176718.540471:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176718.540477:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176718.540482:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176718.540486:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176718.540490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176718.540493:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176718.540496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176718.860445:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306065665, at 4306065728 00000100:00000010:0.0:1786176718.860455:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c960. 00010000:00000001:0.0:1786176718.924471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176718.924476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176718.924484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176718.924490:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176718.924494:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176718.924500:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176718.924508:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176718.924511:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176718.924514:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176718.924516:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176718.924521:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176718.924523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176718.924610:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176718.924613:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176718.924616:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176719.564472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176719.564478:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176719.564480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176719.564485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176719.564488:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176719.564491:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176719.564494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176719.564497:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176719.564499:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176719.948451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176719.948456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176719.948459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176719.948460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176719.948463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176719.948464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176719.948536:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176719.948540:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176719.948543:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176720.077039:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176720.077045:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176720.077721:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176720.077725:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176720.078121:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176720.078122:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176720.078387:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176720.078390:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1786176720.078577:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176720.078579:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:0.0:1786176720.588482:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176720.588492:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176720.588494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176720.588501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176720.588505:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176720.588510:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176720.588512:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176720.588516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176720.588520:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176720.972443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176720.972450:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176720.972454:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176720.972457:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176720.972462:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176720.972465:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176720.972468:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176720.972471:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176720.972476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176720.972479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176720.972482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176720.972483:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176720.972484:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176720.972485:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176720.972486:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176720.972487:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176720.972490:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176720.972491:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176720.972550:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176720.972554:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176720.972558:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176721.612443:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176721.612450:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176721.612452:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176721.612455:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176721.612458:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176721.612461:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176721.612463:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176721.612465:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176721.612468:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176721.996464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176721.996469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176721.996478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176721.996482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176721.996488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176721.996492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176721.996546:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176721.996550:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176721.996553:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176722.636501:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176722.636507:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176722.636509:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176722.636514:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176722.636517:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176722.636520:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176722.636522:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176722.636524:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176722.636526:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176723.020470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176723.020474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176723.020479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176723.020481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176723.020484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176723.020484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176723.020541:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176723.020546:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176723.020550:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176723.660429:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176723.660437:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176723.660440:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176723.660445:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176723.660449:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176723.660453:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176723.660455:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176723.660459:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176723.660462:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176723.980469:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306070784, at 4306070848 00000100:00000010:0.0:1786176723.980485:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c5a0. 00010000:00000001:0.0:1786176724.044451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176724.044456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176724.044460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176724.044465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176724.044471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176724.044475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176724.044534:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176724.044539:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176724.044543:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176724.684476:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176724.684485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176724.684488:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176724.684494:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176724.684498:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176724.684502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176724.684505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176724.684509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176724.684512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176725.068448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176725.068454:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176725.068461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176725.068463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176725.068466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176725.068469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176725.068518:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176725.068522:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176725.068525:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176725.196534:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176725.196549:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176725.196553:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176725.196558:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176725.196564:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176725.196573:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11399 00000400:00000200:3.0:1786176725.196597:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176725.196605:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176725.196610:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176725.196614:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.196619:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176725.196622:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.196625:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176725.196630:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176725.196633:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176725.196636:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176725.196640:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176725.196644:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176725.196647:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176725.196650:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.196652:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176725.196654:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.196657:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176725.196660:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176725.196664:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176725.196667:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176725.196670:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176725.196675:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176725.196679:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.196682:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176725.196685:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176725.196707:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786176725.196712:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176725.196717:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176725.196719:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.196724:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176725.196728:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176725.196734:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176725.196745:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176725.196755:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176725.196784:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235528320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176725.196797:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176725.196802:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176725.196808:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235528320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176725.196841:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176725.196845:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176725.196850:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176725.196852:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176725.196853:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176725.196857:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176725.196859:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11399 00000400:00000200:3.0:1786176725.196864:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176725.196867:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176725.196869:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176725.196872:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.196875:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176725.196882:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.196886:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176725.196889:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176725.196903:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176725.196906:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176725.196909:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176725.196911:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176725.196914:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176725.196919:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.196922:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176725.196925:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.196927:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176725.196930:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176725.196934:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176725.196937:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176725.196939:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176725.196947:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176725.196951:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.196959:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176725.196970:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176725.196976:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.196983:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176725.196995:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786176725.197004:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176725.197022:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176725.197027:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.197033:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176725.197039:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176725.197043:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176725.197052:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176725.197059:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176725.197070:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235528448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176725.197090:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176725.197095:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176725.197102:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235528448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176725.197116:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176725.197123:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176725.197131:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176725.197136:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176725.197144:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176725.197157:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176725.197167:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11399 00000400:00000200:3.0:1786176725.197183:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176725.197200:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176725.197213:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176725.197220:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.197233:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176725.197237:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.197247:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176725.197251:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176725.197253:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176725.197254:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176725.197259:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176725.197266:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176725.197271:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176725.197285:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.197292:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176725.197299:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.197308:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176725.197318:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176725.197323:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176725.197329:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176725.197332:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176725.197342:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176725.197348:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.197352:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176725.197405:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176725.197409:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.197415:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176725.197425:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786176725.197434:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176725.197440:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176725.197444:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.197453:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176725.197459:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:2.1:1786176725.197464:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786176725.197465:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176725.197467:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 02000000:00000010:3.0:1786176725.197475:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176725.197490:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176725.197512:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235528576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176725.197553:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176725.197564:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176725.197582:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235528576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176725.197638:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176725.197653:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176725.197668:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176725.197678:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176725.197682:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176725.197691:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176725.197699:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11399 00000400:00000200:3.0:1786176725.197712:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176725.197722:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176725.197730:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176725.197742:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.197751:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176725.197757:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.197768:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176725.197775:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176725.197779:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176725.197782:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176725.197785:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176725.197788:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176725.197795:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176725.197799:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.197805:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176725.197810:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.197814:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176725.197819:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176725.197826:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176725.197831:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176725.197838:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176725.197846:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176725.197851:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.197852:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176725.197859:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176725.197867:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.197873:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176725.197883:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176725.197887:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176725.197890:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176725.197910:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.197913:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176725.197915:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176725.197919:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176725.197924:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176725.197930:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176725.197938:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235528704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176725.197984:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176725.197992:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176725.198003:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235528704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176725.198060:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176725.198065:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176725.198076:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176725.198079:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176725.198085:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176725.198088:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176725.198094:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11399 00000400:00000200:3.0:1786176725.198100:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176725.198106:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176725.198115:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176725.198119:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.198127:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176725.198131:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.198135:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176725.198141:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176725.198143:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176725.198146:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176725.198151:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176725.198154:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176725.198157:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176725.198163:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.198169:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176725.198176:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176725.198185:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176725.198192:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176725.198201:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176725.198207:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176725.198212:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176725.198216:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176725.198221:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.198227:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176725.198231:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176725.198234:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.198242:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176725.198252:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176725.198256:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176725.198264:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176725.198269:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176725.198272:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176725.198277:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176725.198282:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176725.198291:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176725.198297:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176725.198308:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235528832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176725.198319:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176725.198321:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176725.198327:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235528832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176725.198344:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176725.198355:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11409) 00000800:00000200:2.1:1786176725.198570:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176725.198573:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786176725.198889:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176725.198906:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786176725.199217:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176725.199220:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1786176725.199545:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176725.199549:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000100:00000001:0.0:1786176725.708481:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176725.708488:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176725.708489:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176725.708494:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176725.708498:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176725.708501:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176725.708503:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176725.708505:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176725.708507:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176726.092461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176726.092467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176726.092476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176726.092478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176726.092484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176726.092488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176726.092552:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176726.092556:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176726.092559:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176726.732464:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176726.732474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176726.732477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176726.732483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176726.732488:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176726.732494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176726.732498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176726.732502:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176726.732506:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176727.116477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176727.116483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176727.116493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176727.116497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176727.116503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176727.116507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176727.116580:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176727.116585:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176727.116589:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176727.756452:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176727.756460:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176727.756463:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176727.756469:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176727.756472:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176727.756476:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176727.756478:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176727.756481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176727.756485:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176728.140461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176728.140466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176728.140484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176728.140492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176728.140500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176728.140505:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176728.140580:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176728.140585:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176728.140589:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176728.780490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176728.780500:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176728.780503:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176728.780510:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176728.780515:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176728.780520:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176728.780524:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176728.780529:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176728.780532:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176729.100496:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306075905, at 4306075968 00000100:00000010:0.0:1786176729.100512:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c120. 00010000:00000001:0.0:1786176729.164483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176729.164490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176729.164497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176729.164502:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176729.164505:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176729.164511:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176729.164516:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176729.164519:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176729.164521:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176729.164523:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176729.164526:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176729.164528:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176729.164584:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176729.164588:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176729.164591:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176729.804480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176729.804491:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176729.804494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176729.804501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176729.804504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176729.804509:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176729.804512:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176729.804516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176729.804519:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176730.188539:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176730.188546:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176730.188553:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176730.188554:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176730.188558:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176730.188559:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176730.188680:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176730.188689:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176730.188696:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176730.316564:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176730.316580:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176730.316583:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176730.316589:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176730.316595:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176730.316606:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11404 00000400:00000200:3.0:1786176730.316618:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176730.316626:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176730.316630:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176730.316634:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.316639:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176730.316642:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.316645:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176730.316651:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176730.316654:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176730.316658:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176730.316662:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176730.316665:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176730.316668:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176730.316671:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.316673:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176730.316676:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.316679:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176730.316683:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176730.316687:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176730.316690:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176730.316694:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176730.316699:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176730.316702:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.316705:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176730.316708:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176730.316723:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176730.316727:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176730.316733:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176730.316735:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.316740:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176730.316743:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176730.316750:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176730.316762:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176730.316772:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176730.316785:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235528960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176730.316800:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176730.316807:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176730.316813:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235528960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176730.316869:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176730.316875:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176730.316880:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176730.316882:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176730.316885:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176730.316889:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176730.316893:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11404 00000400:00000200:3.0:1786176730.316900:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176730.316918:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176730.316922:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176730.316926:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.316932:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176730.316937:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.316943:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176730.316955:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176730.316959:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176730.316968:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176730.316975:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176730.316977:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176730.316980:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176730.316991:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.317000:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176730.317005:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.317010:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176730.317019:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176730.317023:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176730.317032:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176730.317039:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176730.317050:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176730.317059:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.317065:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176730.317073:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176730.317088:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.317098:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176730.317105:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176730.317118:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176730.317135:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176730.317138:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.317146:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176730.317152:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176730.317158:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176730.317165:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176730.317181:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176730.317197:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235529088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176730.317263:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176730.317268:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176730.317279:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235529088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176730.317306:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176730.317315:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176730.317320:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176730.317327:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176730.317330:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176730.317338:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176730.317349:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11404 00000400:00000200:3.0:1786176730.317450:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176730.317460:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176730.317470:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176730.317479:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.317494:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176730.317504:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.317512:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176730.317518:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176730.317522:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176730.317529:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176730.317533:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176730.317538:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176730.317554:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176730.317567:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:2.1:1786176730.317567:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176730.317582:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1786176730.317583:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00000040:3.0:1786176730.317592:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.317603:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176730.317612:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176730.317621:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176730.317629:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176730.317640:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176730.317646:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176730.317655:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.317659:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176730.317667:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176730.317673:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.317688:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176730.317704:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176730.317711:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176730.317720:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176730.317724:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.317734:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176730.317739:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176730.317746:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176730.317754:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176730.317763:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176730.317774:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235529216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176730.317803:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176730.317810:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176730.317823:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235529216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176730.317864:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176730.317873:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176730.317885:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176730.317892:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176730.317895:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176730.317902:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176730.317909:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11404 00000400:00000200:3.0:1786176730.317918:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176730.317927:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176730.317932:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176730.317943:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.317955:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176730.317960:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.317971:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176730.317977:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176730.317987:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176730.317991:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176730.317997:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176730.318000:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176730.318007:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176730.318013:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.318022:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176730.318030:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.318040:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176730.318051:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176730.318072:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176730.318088:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176730.318097:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176730.318109:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176730.318118:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.318131:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176730.318140:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176730.318159:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.318171:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176730.318191:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176730.318201:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176730.318216:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176730.318226:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.318241:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176730.318250:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176730.318260:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176730.318279:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176730.318290:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176730.318311:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235529344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176730.318352:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176730.318392:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176730.318406:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235529344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176730.318432:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176730.318437:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176730.318455:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176730.318461:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176730.318468:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176730.318473:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176730.318481:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11404 00000400:00000200:3.0:1786176730.318492:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176730.318509:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176730.318519:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176730.318533:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.318538:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176730.318549:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.318559:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176730.318576:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176730.318586:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176730.318597:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176730.318608:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176730.318622:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176730.318634:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176730.318647:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.318662:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176730.318671:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176730.318679:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176730.318692:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176730.318706:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176730.318721:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176730.318738:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176730.318749:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176730.318754:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.318761:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176730.318764:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176730.318769:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.318775:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176730.318782:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176730.318788:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176730.318795:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176730.318799:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176730.318803:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176730.318807:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176730.318815:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176730.318826:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176730.318837:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176730.318872:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235529472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176730.318919:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176730.318930:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176730.318954:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235529472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1786176730.318990:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176730.318991:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176730.318997:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00000040:3.0:1786176730.319001:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11414) 00000800:00000200:2.1:1786176730.319294:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176730.319295:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176730.319596:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176730.319598:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176730.319811:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176730.319812:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00000001:0.0:1786176730.828498:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176730.828510:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176730.828513:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176730.828519:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176730.828524:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176730.828529:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176730.828532:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176730.828537:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176730.828543:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176731.212500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176731.212509:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176731.212511:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176731.212515:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176731.212520:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176731.212522:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176731.212524:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176731.212526:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176731.212530:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176731.212531:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176731.212533:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176731.212535:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176731.212536:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176731.212537:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176731.212539:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176731.212540:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176731.212541:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176731.212542:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176731.212617:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176731.212623:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176731.212628:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176731.852450:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176731.852458:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176731.852461:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176731.852467:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176731.852472:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176731.852476:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176731.852479:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176731.852481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176731.852485:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176732.236471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176732.236477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176732.236485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176732.236490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176732.236498:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176732.236504:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176732.236621:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176732.236629:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176732.236635:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176733.260442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176733.260446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176733.260451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176733.260454:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176733.260462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176733.260463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176733.260583:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176733.260589:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176733.260593:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176734.220437:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306081025, at 4306081088 00000100:00000010:0.0:1786176734.220457:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c780. 00010000:00000001:0.0:1786176734.284505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176734.284529:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176734.284536:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176734.284538:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176734.284542:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176734.284543:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176734.284707:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176734.284712:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176734.284716:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176734.924524:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176734.924533:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176734.924537:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176734.924543:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176734.924549:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176734.924553:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176734.924556:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176734.924559:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176734.924563:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176735.308542:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176735.308551:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176735.308559:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176735.308560:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176735.308564:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176735.308566:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176735.308681:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176735.308688:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176735.308692:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176735.436573:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176735.436584:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176735.436587:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176735.436591:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176735.436597:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176735.436605:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11409 00000400:00000200:0.0:1786176735.436614:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176735.436621:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176735.436624:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176735.436627:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.436630:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176735.436633:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.436634:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176735.436639:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176735.436641:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176735.436643:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176735.436646:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176735.436647:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176735.436649:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176735.436651:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.436652:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176735.436654:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.436655:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176735.436658:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176735.436661:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176735.436663:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176735.436666:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176735.436668:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176735.436671:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.436672:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176735.436675:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176735.436686:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786176735.436689:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176735.436693:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176735.436694:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.436698:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176735.436702:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176735.436708:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176735.436732:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176735.436741:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176735.436754:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235529600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176735.436765:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176735.436770:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176735.436774:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235529600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176735.436810:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176735.436813:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176735.436816:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176735.436817:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176735.436818:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176735.436821:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176735.436823:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11409 00000400:00000200:0.0:1786176735.436826:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176735.436829:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176735.436831:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176735.436833:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.436835:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176735.436837:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.436839:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176735.436841:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176735.436842:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176735.436843:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176735.436844:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176735.436847:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176735.436849:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176735.436854:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.436857:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176735.436862:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.436864:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176735.436867:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176735.436871:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176735.436876:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176735.436880:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176735.436887:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176735.436893:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.436895:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176735.436901:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176735.436904:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.436908:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176735.436912:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786176735.436915:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176735.436919:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176735.436922:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.436924:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176735.436927:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176735.436932:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176735.436936:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176735.436939:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176735.436945:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235529728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176735.436957:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176735.436961:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176735.436968:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235529728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176735.436976:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176735.436979:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176735.436984:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176735.436985:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176735.436986:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176735.436989:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176735.436993:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11409 00000400:00000200:0.0:1786176735.436997:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176735.437003:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176735.437005:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176735.437010:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437014:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176735.437018:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437022:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176735.437026:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176735.437028:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176735.437031:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176735.437034:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176735.437039:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176735.437042:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176735.437049:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437052:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176735.437056:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437058:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176735.437061:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176735.437062:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437065:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176735.437067:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176735.437071:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176735.437073:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.437074:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176735.437076:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176735.437080:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.437083:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176735.437087:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786176735.437090:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176735.437093:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176735.437094:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.437096:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176735.437099:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176735.437101:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176735.437105:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176735.437108:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176735.437116:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235529856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176735.437125:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176735.437127:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176735.437134:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235529856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176735.437143:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176735.437145:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176735.437152:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176735.437155:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176735.437158:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176735.437163:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176735.437168:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11409 00000400:00000200:0.0:1786176735.437174:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176735.437177:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176735.437181:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176735.437184:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437185:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176735.437190:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437195:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176735.437197:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176735.437199:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176735.437202:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176735.437205:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176735.437209:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000200:2.1:1786176735.437209:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:0.0:1786176735.437214:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000200:2.1:1786176735.437216:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000040:0.0:1786176735.437218:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437221:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176735.437225:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437229:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176735.437231:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176735.437234:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437236:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176735.437238:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176735.437242:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176735.437246:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.437249:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176735.437252:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176735.437255:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.437258:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176735.437263:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786176735.437267:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176735.437270:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176735.437272:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.437276:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176735.437278:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176735.437280:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176735.437285:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176735.437288:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176735.437294:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235529984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176735.437336:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176735.437339:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176735.437348:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235529984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176735.437388:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176735.437392:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176735.437397:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176735.437400:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176735.437402:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176735.437404:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176735.437407:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11409 00000400:00000200:0.0:1786176735.437412:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176735.437415:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176735.437420:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176735.437422:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437424:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176735.437426:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437428:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176735.437431:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176735.437434:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176735.437437:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176735.437440:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176735.437443:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176735.437448:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176735.437453:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437456:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176735.437459:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437463:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176735.437469:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176735.437478:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176735.437482:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176735.437487:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176735.437494:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176735.437500:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.437505:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176735.437509:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176735.437516:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.437520:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176735.437528:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786176735.437533:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176735.437538:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176735.437541:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176735.437546:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176735.437548:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176735.437551:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176735.437556:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176735.437562:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176735.437570:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235530112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176735.437590:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176735.437594:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176735.437599:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235530112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176735.437613:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176735.437619:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11420) 00000800:00000200:2.1:1786176735.437875:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176735.437877:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176735.438072:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176735.438073:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176735.438255:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176735.438257:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176735.438452:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176735.438454:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1786176735.948490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176735.948501:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176735.948504:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176735.948510:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176735.948526:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176735.948530:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176735.948533:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176735.948536:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176735.948539:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176736.332515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176736.332522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176736.332539:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176736.332547:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176736.332559:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176736.332563:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176736.332651:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176736.332657:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176736.332662:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786176736.908458:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786176736.908469:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786176736.908475:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1786176736.908637:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176736.908662:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786176736.908670:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176736.908674:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176736.972533:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176736.972544:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176736.972546:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176736.972552:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176736.972558:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176736.972563:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176736.972565:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176736.972568:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176736.972570:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176737.356431:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176737.356436:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176737.356442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176737.356445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176737.356452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176737.356456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176737.356567:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176737.356574:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176737.356579:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176737.996462:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176737.996471:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176737.996474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176737.996481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176737.996487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176737.996491:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176737.996494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176737.996497:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176737.996500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176738.380475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176738.380481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176738.380488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176738.380491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176738.380498:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176738.380500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176738.380564:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176738.380568:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176738.380570:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176739.020497:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176739.020509:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176739.020512:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176739.020521:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176739.020526:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176739.020532:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176739.020536:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176739.020539:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176739.020544:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176739.340466:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306086145, at 4306086208 00000100:00000010:0.0:1786176739.340480:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c480. 00010000:00000001:0.0:1786176739.404480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176739.404485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176739.404490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176739.404494:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176739.404497:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176739.404500:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176739.404506:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176739.404509:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176739.404511:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176739.404514:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176739.404520:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176739.404524:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176739.404603:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176739.404608:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176739.404613:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176740.044471:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176740.044480:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176740.044484:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176740.044490:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176740.044497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176740.044502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176740.044506:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176740.044509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176740.044512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176740.428471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176740.428475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176740.428484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176740.428488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176740.428492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176740.428494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176740.428603:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176740.428610:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176740.428613:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176740.556473:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176740.556484:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176740.556487:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176740.556492:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176740.556496:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176740.556504:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11415 00000400:00000200:0.0:1786176740.556514:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176740.556520:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176740.556525:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176740.556529:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.556533:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176740.556536:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.556539:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176740.556545:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176740.556547:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176740.556551:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176740.556555:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176740.556558:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176740.556562:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176740.556565:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.556568:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176740.556570:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.556573:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176740.556576:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176740.556579:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176740.556582:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176740.556585:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176740.556591:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176740.556595:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.556598:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176740.556601:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176740.556615:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176740.556620:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176740.556626:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176740.556628:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.556632:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176740.556635:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176740.556640:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176740.556648:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176740.556655:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176740.556665:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235530240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176740.556675:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176740.556679:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176740.556683:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235530240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176740.556706:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176740.556709:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176740.556712:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176740.556713:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176740.556714:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176740.556717:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176740.556719:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11415 00000400:00000200:0.0:1786176740.556722:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176740.556725:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176740.556726:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176740.556729:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.556730:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176740.556732:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.556734:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176740.556735:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176740.556737:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176740.556738:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176740.556740:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176740.556741:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176740.556742:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176740.556758:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.556759:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176740.556761:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.556762:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176740.556763:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176740.556765:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176740.556768:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176740.556769:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176740.556775:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176740.556776:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.556778:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176740.556783:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176740.556785:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.556787:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176740.556791:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176740.556794:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176740.556800:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176740.556803:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.556811:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176740.556820:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176740.556825:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176740.556835:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786176740.556848:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176740.556862:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235530368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176740.556898:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176740.556908:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176740.556921:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235530368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176740.556945:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176740.556949:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176740.556953:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176740.556956:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176740.556964:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176740.556976:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176740.556987:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11415 00000400:00000200:0.0:1786176740.557001:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176740.557011:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176740.557018:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176740.557027:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.557038:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176740.557047:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.557061:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176740.557068:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176740.557072:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176740.557083:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176740.557087:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176740.557089:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176740.557097:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176740.557113:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.557119:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176740.557126:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.557130:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176740.557136:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176740.557143:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176740.557151:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176740.557158:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176740.557162:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176740.557168:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.557170:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176740.557171:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176740.557174:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.557183:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176740.557189:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176740.557200:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176740.557205:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176740.557209:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.557219:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176740.557228:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176740.557235:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176740.557246:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176740.557256:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176740.557263:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00100000:0.0:1786176740.557270:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235530496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1786176740.557270:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:0.0:1786176740.557300:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176740.557309:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176740.557316:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235530496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176740.557337:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176740.557347:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176740.557355:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176740.557401:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176740.557403:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176740.557413:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176740.557421:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11415 00000400:00000200:0.0:1786176740.557435:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176740.557449:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176740.557459:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176740.557470:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.557480:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176740.557495:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.557510:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176740.557517:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176740.557524:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176740.557531:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176740.557539:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176740.557545:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176740.557552:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176740.557568:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.557578:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176740.557594:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.557603:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176740.557617:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176740.557632:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176740.557641:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176740.557655:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176740.557661:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176740.557671:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.557681:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176740.557687:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176740.557694:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.557703:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176740.557710:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176740.557726:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176740.557734:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176740.557739:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.557759:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176740.557764:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176740.557768:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176740.557780:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176740.557785:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176740.557795:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235530624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176740.557828:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176740.557836:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176740.557850:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235530624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176740.557907:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176740.557921:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176740.557933:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176740.557939:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176740.557944:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176740.557950:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176740.557959:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11415 00000400:00000200:0.0:1786176740.557968:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176740.557980:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176740.557984:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176740.557992:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.558001:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176740.558004:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.558009:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176740.558016:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176740.558018:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176740.558028:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176740.558036:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176740.558051:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176740.558055:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176740.558059:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.558070:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176740.558079:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176740.558087:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176740.558097:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176740.558104:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176740.558113:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176740.558121:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176740.558130:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176740.558138:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.558151:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176740.558155:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176740.558171:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.558187:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176740.558195:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176740.558200:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176740.558209:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176740.558214:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176740.558221:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176740.558228:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176740.558233:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176740.558244:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176740.558257:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176740.558269:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235530752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176740.558309:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176740.558315:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176740.558327:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235530752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176740.558350:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176740.558383:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11425) 00000800:00000200:2.1:1786176740.558421:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176740.558424:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176740.558617:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176740.558618:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176740.558814:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176740.558816:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786176740.559060:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176740.559061:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1786176741.068473:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176741.068483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176741.068487:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176741.068493:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176741.068497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176741.068501:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176741.068503:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176741.068506:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176741.068508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176741.452485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176741.452497:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176741.452503:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176741.452509:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176741.452517:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176741.452522:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176741.452526:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176741.452530:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176741.452544:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176741.452549:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176741.452553:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176741.452559:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176741.452561:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176741.452567:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176741.452570:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176741.452571:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176741.452572:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176741.452574:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176741.452634:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176741.452638:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176741.452642:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176742.092431:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176742.092439:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176742.092441:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176742.092445:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176742.092448:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176742.092451:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176742.092454:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176742.092456:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176742.092459:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176742.476455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176742.476460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176742.476467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176742.476468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176742.476478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176742.476480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176742.476542:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176742.476545:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176742.476547:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176743.116503:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176743.116515:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176743.116518:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176743.116527:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176743.116533:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176743.116537:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176743.116541:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176743.116545:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176743.116549:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176743.500474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176743.500478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176743.500484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176743.500488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176743.500493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176743.500494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176743.500595:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176743.500603:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176743.500609:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176744.140425:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176744.140436:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176744.140439:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176744.140447:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176744.140452:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176744.140457:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176744.140460:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176744.140464:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176744.140467:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176744.460440:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306091265, at 4306091328 00000100:00000010:0.0:1786176744.460448:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290ca80. 00010000:00000001:0.0:1786176744.524450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176744.524456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176744.524466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176744.524471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176744.524489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176744.524491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176744.524541:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176744.524545:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176744.524548:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176745.164420:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176745.164428:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176745.164430:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176745.164435:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176745.164439:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176745.164442:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176745.164444:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176745.164446:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176745.164449:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176745.548438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176745.548447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176745.548454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176745.548455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176745.548459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176745.548460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176745.548517:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176745.548521:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176745.548525:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176745.677087:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176745.677094:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176745.677915:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176745.677918:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176745.678223:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176745.678226:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176745.678542:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176745.678544:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176745.678963:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176745.678966:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1786176746.188409:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176746.188418:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176746.188421:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176746.188428:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176746.188433:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176746.188439:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176746.188443:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176746.188447:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176746.188451:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176746.572422:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176746.572425:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176746.572431:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176746.572432:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176746.572436:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176746.572437:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176746.572490:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176746.572495:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176746.572498:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176747.212425:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176747.212431:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176747.212433:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176747.212438:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176747.212441:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176747.212444:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176747.212447:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176747.212450:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176747.212453:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176747.596427:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176747.596433:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176747.596438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176747.596439:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176747.596442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176747.596443:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176747.596485:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176747.596489:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176747.596492:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176748.236407:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176748.236415:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176748.236417:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176748.236422:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176748.236425:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176748.236428:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176748.236431:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176748.236433:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176748.236436:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176748.620414:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176748.620420:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176748.620425:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176748.620428:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176748.620430:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176748.620433:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176748.620438:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176748.620440:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176748.620444:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176748.620449:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176748.620453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176748.620455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176748.620504:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176748.620509:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176748.620513:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176749.580450:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306096384, at 4306096448 00000100:00000010:0.0:1786176749.580469:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c626c0. 00010000:00000001:0.0:1786176749.644467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176749.644475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176749.644481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176749.644484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176749.644488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176749.644489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176749.644562:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176749.644568:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176749.644573:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176750.668437:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176750.668441:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176750.668450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176750.668453:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176750.668458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176750.668461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176750.668511:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176750.668516:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176750.668520:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176750.796920:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176750.796925:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176750.797641:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176750.797643:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176750.797910:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176750.797912:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176750.798122:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176750.798124:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176750.798391:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176750.798393:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1786176751.692459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176751.692465:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176751.692469:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176751.692480:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176751.692486:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176751.692489:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176751.692490:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176751.692492:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176751.692500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176751.692503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176751.692508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176751.692511:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176751.692512:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176751.692513:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176751.692514:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176751.692515:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176751.692516:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176751.692517:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176751.692578:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176751.692581:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176751.692584:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176752.716455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176752.716459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176752.716465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176752.716469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176752.716473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176752.716475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176752.716542:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176752.716547:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176752.716551:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176753.740502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176753.740511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176753.740518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176753.740520:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176753.740523:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176753.740525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176753.740601:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176753.740606:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176753.740610:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176754.380787:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176754.380796:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176754.380799:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176754.380805:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176754.380810:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176754.380814:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176754.380817:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176754.380820:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176754.380824:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176754.700448:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306101504, at 4306101568 00000100:00000010:0.0:1786176754.700464:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62540. 00010000:00000001:0.0:1786176754.764447:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176754.764452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176754.764458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176754.764459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176754.764463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176754.764464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176754.764517:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176754.764520:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176754.764523:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176755.404955:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176755.404968:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176755.404972:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176755.404980:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176755.404986:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176755.404992:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176755.404996:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176755.405001:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176755.405006:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176755.788525:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176755.788530:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176755.788537:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176755.788540:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176755.788547:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176755.788550:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176755.788656:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176755.788662:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176755.788667:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176755.916972:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176755.916978:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176755.917714:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176755.917717:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176755.917922:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176755.917923:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176755.918134:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176755.918136:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786176755.918435:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176755.918437:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1786176756.812446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176756.812452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176756.812460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176756.812463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176756.812468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176756.812472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176756.812568:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176756.812575:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176756.812581:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176757.836524:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176757.836532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176757.836540:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176757.836542:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176757.836546:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176757.836548:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176757.836639:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176757.836643:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176757.836652:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176758.860442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176758.860446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176758.860452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176758.860455:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176758.860458:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176758.860462:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176758.860466:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176758.860468:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176758.860472:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176758.860475:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176758.860478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176758.860479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176758.860535:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176758.860538:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176758.860540:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176759.820498:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306106624, at 4306106688 00000100:00000010:0.0:1786176759.820519:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c240. 00010000:00000001:0.0:1786176759.884501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176759.884508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176759.884517:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176759.884521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176759.884529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176759.884531:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176759.884618:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176759.884624:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176759.884628:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176760.908428:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176760.908433:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176760.908442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176760.908448:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176760.908457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176760.908460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176760.908523:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176760.908527:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176760.908530:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176761.037033:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176761.037040:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176761.037797:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176761.037800:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176761.038124:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176761.038127:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176761.038449:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176761.038452:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176761.038763:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176761.038766:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1786176761.932467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176761.932473:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176761.932477:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176761.932481:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176761.932485:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176761.932488:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176761.932493:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176761.932497:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176761.932502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176761.932505:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176761.932509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176761.932510:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176761.932511:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176761.932512:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176761.932514:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176761.932514:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176761.932516:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176761.932517:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176761.932582:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176761.932585:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176761.932588:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176762.956454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176762.956461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176762.956470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176762.956474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176762.956481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176762.956485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176762.956584:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176762.956588:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176762.956591:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176763.980509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176763.980529:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176763.980542:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176763.980549:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176763.980562:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176763.980565:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176763.980634:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176763.980639:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176763.980642:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176764.940430:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306111744, at 4306111808 00000100:00000010:0.0:1786176764.940445:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62840. 00010000:00000001:0.0:1786176765.004579:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176765.004583:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176765.004588:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176765.004589:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176765.004594:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176765.004595:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176765.004673:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176765.004679:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176765.004683:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176766.028436:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176766.028442:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176766.028448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176766.028451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176766.028458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176766.028462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176766.028529:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176766.028533:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176766.028535:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1786176766.157105:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176766.157114:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176766.157724:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176766.157726:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176766.158029:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176766.158032:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176766.158328:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176766.158330:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176766.158670:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176766.158672:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00010000:00000001:0.0:1786176767.052464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176767.052469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176767.052474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176767.052477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176767.052484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176767.052510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176767.052589:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176767.052595:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176767.052599:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176768.076454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176768.076462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176768.076470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176768.076471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176768.076475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176768.076490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176768.076549:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176768.076555:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176768.076559:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176769.100467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176769.100485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176769.100490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176769.100495:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176769.100499:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176769.100503:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176769.100507:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176769.100510:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176769.100514:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176769.100516:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176769.100522:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176769.100525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176769.100584:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176769.100589:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176769.100592:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176770.060498:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306116864, at 4306116928 00000100:00000010:0.0:1786176770.060509:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62c60. 00010000:00000001:0.0:1786176770.124434:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176770.124441:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176770.124450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176770.124453:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176770.124472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176770.124474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176770.124560:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176770.124566:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176770.124570:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176770.764558:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176770.764568:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176770.764572:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176770.764579:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176770.764585:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176770.764588:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176770.764591:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176770.764593:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176770.764596:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176771.148489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176771.148496:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176771.148501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176771.148505:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176771.148510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176771.148516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176771.148604:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176771.148609:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176771.148613:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176771.277344:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176771.278071:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176771.278247:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176771.278494:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176771.278651:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176771.788481:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176771.788496:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176771.788499:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176771.788505:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176771.788508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176771.788512:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176771.788515:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176771.788517:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176771.788520:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176772.172487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176772.172497:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176772.172504:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176772.172512:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176772.172521:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176772.172525:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176772.172530:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176772.172535:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176772.172542:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176772.172545:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176772.172553:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176772.172555:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176772.172556:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176772.172559:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176772.172560:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176772.172561:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176772.172563:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176772.172564:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176772.172630:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176772.172634:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176772.172637:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176772.812533:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176772.812544:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176772.812546:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176772.812553:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176772.812560:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176772.812566:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176772.812572:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176772.812576:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176772.812581:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176773.196502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176773.196514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176773.196523:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176773.196527:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176773.196532:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176773.196535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176773.196604:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176773.196609:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176773.196612:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176773.836554:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176773.836571:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176773.836574:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176773.836581:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176773.836586:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176773.836591:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176773.836594:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176773.836597:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176773.836600:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176774.220487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176774.220491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176774.220497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176774.220498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176774.220502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176774.220503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176774.220563:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176774.220570:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176774.220573:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1786176774.348702:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:0.0:1786176774.348738:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1786176774.348746:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176774.348749:0:2380:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:0.0:1786176774.348754:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176774.348757:0:2380:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1786176774.348761:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176774.348765:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d300. 00000400:00000200:0.0:1786176774.348769:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176774.348772:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176774.348777:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28dd0 00000400:00000010:0.0:1786176774.348779:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28dd0. 00000100:00000001:0.0:1786176774.348783:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176774.348791:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328bb100 x1872930235534080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176774.348817:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176774.348822:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb100 x1872930235534080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176774.348831:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176774.348834:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176774.348845:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176774.348848:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6df00. 00000400:00000200:0.0:1786176774.348850:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176774.348854:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176774.348858:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:0.0:1786176774.348860:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:0.0:1786176774.348863:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176774.348869:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328b9f80 x1872930235534208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176774.348879:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176774.348884:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235534208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176774.348894:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176774.348899:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176774.348910:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176774.348913:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d100. 00000400:00000200:0.0:1786176774.348920:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176774.348928:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176774.348935:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28880 00000400:00000010:0.0:1786176774.348938:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28880. 00000100:00000001:0.0:1786176774.348941:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176774.348949:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328bb480 x1872930235534336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176774.348963:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176774.348970:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb480 x1872930235534336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176774.348983:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176774.348988:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176774.348998:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176774.349003:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6db00. 00000400:00000200:0.0:1786176774.349009:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176774.349026:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176774.349032:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28330 00000400:00000010:0.0:1786176774.349036:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28330. 00000100:00000001:0.0:1786176774.349042:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176774.349051:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328ba680 x1872930235534464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176774.349071:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176774.349078:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328ba680 x1872930235534464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176774.349105:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176774.349110:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176774.349120:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176774.349126:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d900. 00000400:00000200:0.0:1786176774.349133:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176774.349143:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176774.349152:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28aa0 00000400:00000010:0.0:1786176774.349157:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28aa0. 00000100:00000001:0.0:1786176774.349164:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176774.349174:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328b9c00 x1872930235534592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176774.349204:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176774.349211:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9c00 x1872930235534592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176796 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176774.349223:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176774.349228:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:0.0:1786176774.796489:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786176774.796496:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786176774.796501:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1786176774.796654:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176774.796669:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786176774.796675:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176774.796679:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176774.860492:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176774.860501:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176774.860504:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176774.860511:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176774.860516:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176774.860521:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176774.860524:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176774.860527:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176774.860531:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176775.180464:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306121985, at 4306122048 00000100:00000010:0.0:1786176775.180491:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c6c0. 00010000:00000001:0.0:1786176775.244465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176775.244471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176775.244479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176775.244483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176775.244487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176775.244489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176775.244534:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176775.244537:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176775.244540:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176775.884466:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176775.884474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176775.884477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176775.884483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176775.884489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176775.884493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176775.884497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176775.884500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176775.884503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176776.268527:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176776.268535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176776.268541:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176776.268543:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176776.268547:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176776.268548:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176776.268618:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176776.268631:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176776.268634:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1786176776.396936:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176776.396941:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176776.397501:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176776.397504:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176776.397838:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176776.397841:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176776.398155:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176776.398158:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786176776.398554:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176776.398557:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786176776.908479:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176776.908488:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176776.908490:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176776.908497:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176776.908502:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176776.908506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176776.908509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176776.908512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176776.908516:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176777.292458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176777.292463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176777.292471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176777.292475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176777.292479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176777.292481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176777.292538:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176777.292542:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176777.292545:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176777.932455:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176777.932463:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176777.932466:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176777.932473:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176777.932478:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176777.932481:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176777.932483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176777.932486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176777.932489:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176778.316493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176778.316497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176778.316502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176778.316504:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176778.316507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176778.316509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176778.316569:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176778.316572:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176778.316575:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176778.956538:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176778.956551:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176778.956554:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176778.956562:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176778.956568:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176778.956574:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176778.956577:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176778.956580:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176778.956584:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176779.340480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176779.340485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176779.340491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176779.340496:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176779.340499:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176779.340503:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176779.340506:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176779.340518:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176779.340524:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176779.340527:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176779.340532:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176779.340533:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176779.340629:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176779.340637:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176779.340642:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176779.980458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176779.980466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176779.980468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176779.980474:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176779.980479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176779.980483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176779.980486:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176779.980489:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176779.980492:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176780.300471:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306127104, at 4306127168 00000100:00000010:0.0:1786176780.300488:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cc00. 00010000:00000001:0.0:1786176780.364481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176780.364485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176780.364491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176780.364492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176780.364496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176780.364498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176780.364557:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176780.364562:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176780.364566:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176781.004469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176781.004478:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176781.004481:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176781.004486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176781.004491:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176781.004496:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176781.004500:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176781.004503:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176781.004507:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176781.388460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176781.388465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176781.388470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176781.388472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176781.388478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176781.388480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176781.388543:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176781.388547:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176781.388550:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176781.516498:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176781.516507:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176781.516509:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176781.516513:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176781.516517:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176781.516524:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11455 00000400:00000200:0.0:1786176781.516532:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176781.516539:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176781.516543:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176781.516546:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.516550:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176781.516553:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.516556:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176781.516560:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176781.516563:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176781.516566:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176781.516570:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176781.516573:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176781.516576:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176781.516579:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.516581:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176781.516583:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.516585:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176781.516589:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176781.516602:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176781.516606:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176781.516609:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176781.516612:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176781.516614:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.516615:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176781.516618:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176781.516626:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176781.516629:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176781.516632:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176781.516633:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.516636:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176781.516638:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176781.516643:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176781.516650:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176781.516655:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176781.516663:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235535360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176781.516671:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176781.516674:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176781.516678:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235535360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176781.516708:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176781.516712:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176781.516716:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176781.516718:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176781.516719:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176781.516723:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176781.516726:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11455 00000400:00000200:0.0:1786176781.516730:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176781.516733:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176781.516735:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176781.516739:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.516741:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176781.516743:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.516745:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176781.516748:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176781.516749:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176781.516751:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176781.516754:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176781.516756:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176781.516760:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176781.516764:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.516766:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176781.516769:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.516771:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176781.516773:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176781.516790:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176781.516799:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176781.516806:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176781.516824:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176781.516830:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.516836:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176781.516843:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176781.516849:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.516854:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176781.516861:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176781.516866:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176781.516871:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176781.516875:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.516881:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176781.516884:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176781.516887:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176781.516894:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176781.516900:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176781.516909:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235535488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176781.516922:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176781.516925:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176781.516930:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235535488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176781.516939:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176781.516944:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176781.516950:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176781.516953:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176781.516956:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176781.516959:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176781.516966:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11455 00000400:00000200:0.0:1786176781.516976:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176781.516983:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176781.516986:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176781.516994:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.516998:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176781.517003:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517007:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176781.517011:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176781.517013:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176781.517016:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176781.517019:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176781.517022:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176781.517025:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176781.517031:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517035:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1786176781.517036:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:0.0:1786176781.517040:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:2.1:1786176781.517042:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000100:00000001:0.0:1786176781.517044:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176781.517049:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176781.517055:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517058:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176781.517060:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176781.517064:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176781.517068:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.517071:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176781.517073:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176781.517078:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.517081:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176781.517084:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176781.517090:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176781.517096:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176781.517100:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.517104:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176781.517108:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176781.517113:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176781.517120:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176781.517128:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176781.517140:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176781.517153:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176781.517156:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176781.517164:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176781.517179:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176781.517183:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176781.517189:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176781.517192:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176781.517195:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176781.517200:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176781.517204:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11455 00000400:00000200:0.0:1786176781.517213:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176781.517218:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176781.517221:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176781.517228:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517233:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176781.517237:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517241:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176781.517244:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176781.517247:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176781.517250:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176781.517253:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176781.517258:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176781.517262:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176781.517268:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517271:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176781.517277:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517281:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176781.517286:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176781.517292:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517296:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176781.517301:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176781.517305:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176781.517310:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.517315:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176781.517319:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176781.517324:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.517328:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176781.517333:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176781.517338:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176781.517342:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176781.517343:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.517391:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176781.517393:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176781.517395:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176781.517398:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176781.517401:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176781.517408:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176781.517430:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176781.517435:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176781.517441:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176781.517453:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176781.517459:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176781.517464:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176781.517467:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176781.517471:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176781.517475:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176781.517481:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11455 00000400:00000200:0.0:1786176781.517487:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176781.517493:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176781.517497:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176781.517504:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517509:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176781.517514:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517517:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176781.517519:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176781.517523:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176781.517527:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176781.517530:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176781.517534:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176781.517538:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176781.517546:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517550:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176781.517555:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517559:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176781.517565:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176781.517570:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176781.517575:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176781.517580:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176781.517584:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176781.517588:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.517591:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176781.517594:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176781.517599:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.517601:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176781.517607:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176781.517611:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176781.517614:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176781.517617:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176781.517619:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176781.517623:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176781.517626:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176781.517631:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176781.517635:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176781.517644:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176781.517658:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176781.517663:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176781.517672:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176781.517695:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176781.517705:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11466) 00000800:00000200:2.1:1786176781.517734:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176781.517736:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000100:00000001:2.0:1786176781.517824:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.517825:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176781.517826:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.517827:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176781.517829:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176781.517831:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176781.517833:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176781.517834:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176781.517837:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314a80 x1872930235535488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176781.517841:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235535488:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176781.517843:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176781.517846:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.517850:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176781.517851:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176781.517852:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176781.517853:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176781.517855:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176781.517856:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235535616:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176781.517858:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176781.517859:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176781.517860:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176781.517861:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176781.517863:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786176781.517865:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176781.517867:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176781.517868:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176781.517870:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176781.517872:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235535616, portal 10 00000100:00000001:2.0:1786176781.517873:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176781.517875:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794203392 : -110371915348224 : ffff9b9e09314700) 00000100:00000040:2.0:1786176781.517878:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.517883:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176781.517884:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176781.517886:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176781.517887:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235535616, offset 0 00000400:00000200:2.0:1786176781.517890:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176781.517892:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176781.517894:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176781.517896:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176781.517898:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176781.517899:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176781.517901:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176781.517903:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426251:426251:256:4294967295] 192.168.202.132@tcp LPNI seq info [426213:426213:8:4294967295] 00000400:00000200:2.0:1786176781.517906:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176781.517910:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176781.517912:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176781.517914:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176781.517916:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176781.517918:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786176781.517920:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176781.517922:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786176781.517924:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176781.517926:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176781.517928:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176781.517930:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176781.517932:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176781.517933:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176781.517935:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176781.517937:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.517946:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176781.517948:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.517952:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176781.517953:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176781.517955:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176781.517956:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786176781.517958:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176781.517959:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306128388 00000100:00000001:2.0:1786176781.517960:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306132224 00000100:00000001:2.0:1786176781.517962:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306128388 00000100:00000001:2.0:1786176781.517964:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306132224 00000100:00000001:2.0:1786176781.517965:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176781.517966:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176781.517967:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176781.517969:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176781/real 1786176781] req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176781.517974:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176781.517975:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176781.517975:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176781.517977:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176781.517978:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176781.517979:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176781.517981:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176781.517986:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.517990:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176781.517991:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.517992:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176781.517993:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176781.517996:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518000:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176781.518002:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518006:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176781.518007:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518008:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176781.518009:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518010:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176781.518012:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176781.518013:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176781.518015:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176781.518016:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176781.518018:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176781.518023:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235535616:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176781.518025:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176781.518027:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518031:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176781.518032:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176781.518033:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176781.518034:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176781.518035:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176781.518037:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235535744:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176781.518039:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176781.518040:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176781.518041:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176781.518043:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176781.518044:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9000. 02000000:00000001:2.0:1786176781.518046:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176781.518047:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176781.518049:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176781.518051:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176781.518053:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235535744, portal 4 00000100:00000001:2.0:1786176781.518054:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176781.518055:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215040 : -110371915336576 : ffff9b9e09317480) 00000100:00000040:2.0:1786176781.518059:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518062:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176781.518064:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176781.518065:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176781.518067:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235535744, offset 0 00000400:00000200:2.0:1786176781.518069:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176781.518071:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176781.518072:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176781.518074:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176781.518076:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176781.518077:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176781.518079:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176781.518081:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426252:426252:256:4294967295] 192.168.202.132@tcp LPNI seq info [426214:426214:8:4294967295] 00000400:00000200:2.0:1786176781.518083:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176781.518087:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176781.518090:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176781.518091:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176781.518094:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176781.518096:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786176781.518098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176781.518100:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786176781.518102:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176781.518104:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176781.518105:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176781.518108:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176781.518109:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176781.518110:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176781.518112:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176781.518114:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518119:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176781.518121:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518125:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176781.518126:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176781.518127:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176781.518129:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786176781.518130:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176781.518132:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306128388 00000100:00000001:2.0:1786176781.518133:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306132224 00000100:00000001:2.0:1786176781.518134:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306128388 00000100:00000001:2.0:1786176781.518135:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306132224 00000100:00000001:2.0:1786176781.518137:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176781.518138:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176781.518139:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176781.518141:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176781/real 1786176781] req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176781.518146:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176781.518147:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176781.518148:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176781.518149:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176781.518150:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176781.518152:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176781.518154:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176781.518159:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518162:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176781.518163:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518164:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176781.518165:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176781.518167:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518171:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176781.518173:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518177:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176781.518178:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518179:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176781.518180:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518181:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176781.518182:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176781.518183:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176781.518185:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176781.518186:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176781.518189:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176781.518198:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235535744:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176781.518201:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518202:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176781.518204:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314a80 x1872930235535488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518208:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176781.518209:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176781.518210:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176781.518211:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176781.518212:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176781.518213:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176781.518214:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176781.518215:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:2.0:1786176781.518216:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176781.518217:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176781.518218:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314a80. 00000100:00000001:2.0:1786176781.518220:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176781.518221:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518222:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176781.518224:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314700 x1872930235535616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518228:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176781.518229:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176781.518230:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786176781.518231:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176781.518231:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176781.518232:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176781.518233:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176781.518234:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:2.0:1786176781.518235:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176781.518236:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176781.518237:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314700. 00000100:00000001:2.0:1786176781.518239:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176781.518240:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518241:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176781.518243:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317480 x1872930235535744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518247:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176781.518248:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176781.518249:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:2.0:1786176781.518250:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176781.518250:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176781.518251:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176781.518252:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176781.518253:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:2.0:1786176781.518254:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176781.518255:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176781.518255:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317480. 00000100:00000001:2.0:1786176781.518257:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176781.518258:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518259:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518261:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176781.518261:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176781.518263:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176781.518263:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176781.518264:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176781.518266:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518270:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176781.518271:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176781.518272:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176781.518273:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176781.518274:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176781.518276:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235535872:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176781.518278:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176781.518279:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176781.518280:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176781.518281:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176781.518283:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9000. 02000000:00000001:2.0:1786176781.518285:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176781.518287:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176781.518288:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176781.518290:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176781.518292:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235535872, portal 4 00000100:00000001:2.0:1786176781.518293:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176781.518294:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794213248 : -110371915338368 : ffff9b9e09316d80) 00000100:00000040:2.0:1786176781.518298:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518302:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176781.518303:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176781.518305:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176781.518306:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235535872, offset 0 00000400:00000200:2.0:1786176781.518308:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176781.518310:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176781.518312:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176781.518314:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176781.518315:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176781.518317:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176781.518318:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176781.518320:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426253:426253:256:4294967295] 192.168.202.132@tcp LPNI seq info [426215:426215:8:4294967295] 00000400:00000200:2.0:1786176781.518323:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176781.518327:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176781.518329:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176781.518332:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176781.518335:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176781.518338:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786176781.518342:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176781.518345:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786176781.518378:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176781.518380:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176781.518384:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176781.518387:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176781.518390:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176781.518391:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176781.518393:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176781.518396:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518401:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176781.518403:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176806 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518407:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176781.518409:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176781.518411:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176781.518413:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786176781.518415:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176781.518416:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306128391 00000100:00000001:2.0:1786176781.518418:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306132224 00000100:00000001:2.0:1786176781.518420:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306128391 00000100:00000001:2.0:1786176781.518421:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306132224 00000100:00000001:2.0:1786176781.518422:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176781.518423:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176781.518425:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176781.518427:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176781/real 1786176781] req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176781.518432:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176781.518433:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176781.518434:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176781.518436:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176781.518437:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176781.518438:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176781.518441:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176781.518446:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518450:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176781.518451:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518452:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176781.518453:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176781.518455:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518459:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176781.518461:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518465:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176781.518467:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518467:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176781.518468:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518469:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176781.518470:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176781.518472:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176781.518474:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176781.518475:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176781.518478:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176781.518483:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235535872:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176781.518485:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518486:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176781.518488:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316d80 x1872930235535872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176806 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176781.518493:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176781.518494:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176781.518495:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:2.0:1786176781.518497:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176781.518498:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176781.518498:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176781.518500:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176781.518501:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:2.0:1786176781.518502:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176781.518503:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176781.518504:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316d80. 00000100:00000001:2.0:1786176781.518506:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176781.518507:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518508:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176781.518510:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176781.518511:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176781.518512:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176781.518514:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176782.028477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176782.028497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176782.028500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176782.028507:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176782.028512:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176782.028516:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176782.028518:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176782.028521:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176782.028524:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176782.412503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176782.412517:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176782.412523:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176782.412532:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176782.412541:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176782.412546:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176782.412549:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176782.412551:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176782.412557:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176782.412558:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176782.412564:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176782.412566:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176782.412568:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176782.412569:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176782.412571:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176782.412573:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176782.412583:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176782.412585:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176782.412672:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176782.412677:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176782.412681:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176783.052523:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176783.052537:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176783.052541:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176783.052551:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176783.052559:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176783.052566:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176783.052570:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176783.052573:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176783.052579:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176783.436465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176783.436470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176783.436479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176783.436483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176783.436487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176783.436491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176783.436565:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176783.436570:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176783.436573:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176784.076541:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176784.076552:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176784.076554:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176784.076560:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176784.076564:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176784.076580:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176784.076585:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176784.076589:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176784.076593:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176784.460476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176784.460482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176784.460492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176784.460496:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176784.460503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176784.460506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176784.460568:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176784.460573:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176784.460576:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176785.100484:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176785.100492:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176785.100496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176785.100502:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176785.100506:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176785.100511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176785.100517:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176785.100521:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176785.100525:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176785.420467:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306132224, at 4306132288 00000100:00000010:0.0:1786176785.420478:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cf60. 00010000:00000001:0.0:1786176785.484504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176785.484510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176785.484522:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176785.484527:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176785.484535:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176785.484539:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176785.484637:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176785.484645:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176785.484650:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176786.124453:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176786.124462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176786.124465:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176786.124470:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176786.124484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176786.124488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176786.124490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176786.124493:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176786.124496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176786.508507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176786.508516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176786.508524:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176786.508526:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176786.508537:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176786.508539:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176786.508618:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176786.508622:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176786.508625:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176786.636474:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176786.636486:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176786.636488:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176786.636493:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176786.636497:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176786.636505:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11461 00000400:00000200:0.0:1786176786.636514:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176786.636520:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176786.636524:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176786.636528:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636531:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176786.636533:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636535:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176786.636539:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176786.636541:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176786.636544:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176786.636547:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176786.636549:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176786.636550:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176786.636552:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636554:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176786.636555:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636557:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176786.636560:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176786.636562:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636564:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176786.636566:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176786.636569:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176786.636571:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176786.636573:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176786.636575:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176786.636586:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786176786.636590:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176786.636594:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176786.636596:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176786.636601:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176786.636604:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176786.636609:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176786.636620:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176786.636629:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176786.636651:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176786.636663:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176786.636667:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176786.636671:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176786.636692:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176786.636695:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176786.636698:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176786.636699:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176786.636700:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176786.636703:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176786.636705:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11461 00000400:00000200:0.0:1786176786.636708:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176786.636711:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176786.636712:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176786.636714:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636716:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176786.636718:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636719:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176786.636722:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176786.636723:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176786.636725:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176786.636727:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176786.636728:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176786.636730:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176786.636732:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636734:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176786.636735:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636737:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176786.636740:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176786.636742:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636745:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176786.636746:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176786.636753:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176786.636755:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176786.636756:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176786.636765:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176786.636770:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.636771:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786176786.636773:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176786.636778:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786176786.636781:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000100:00000001:2.0:1786176786.636782:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1786176786.636784:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176786.636786:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.636787:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1786176786.636789:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176786.636794:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176786.636797:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176786.636800:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786176786.636805:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176786.636808:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786176786.636814:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176786.636858:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176786.636866:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.636872:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1786176786.636874:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.636876:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176786.636889:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1786176786.636894:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176786.636898:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176786.636900:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176786.636902:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1786176786.636902:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1786176786.636904:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176786.636908:0:40394:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1786176786.636911:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176786.636913:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11461 00000100:00100000:2.0:1786176786.636922:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235536000:192.168.202.132@tcp:250:kworker.0 00000400:00000200:0.0:1786176786.636923:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176786.636932:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786176786.636934:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1786176786.636938:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176786.636942:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636944:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1786176786.636944:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1786176786.636952:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636954:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:2.0:1786176786.636954:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176786.636961:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176786.636963:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176786.636965:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:2.0:1786176786.636965:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1786176786.636970:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176786.636972:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176786.636974:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176786.636977:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.636978:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176786.636980:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000010:2.0:1786176786.636980:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:0.0:1786176786.636987:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:2.0:1786176786.636989:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1786176786.636990:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176786.636998:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637000:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176786.637007:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:0.0:1786176786.637008:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176786.637015:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1786176786.637016:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 02000000:00000001:0.0:1786176786.637028:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176786.637031:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:2.0:1786176786.637033:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000008:00100000:0.0:1786176786.637036:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000200:2.0:1786176786.637040:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235536000, portal 25 00000008:00000001:0.0:1786176786.637041:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176786.637052:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637053:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176786.637058:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794208768 : -110371915342848 : ffff9b9e09315c00) 00000100:00000010:0.0:1786176786.637061:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786176786.637065:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000040:2.0:1786176786.637071:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786176786.637072:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176786.637079:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176786.637086:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176786.637092:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176786.637099:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637100:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:0.0:1786176786.637108:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000040:2.0:1786176786.637109:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637118:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786176786.637118:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176786.637134:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235536000, offset 0 00000100:00100000:0.0:1786176786.637141:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786176786.637151:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637153:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176786.637158:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176786.637163:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786176786.637174:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786176786.637177:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176786.637182:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176786.637185:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:2.0:1786176786.637189:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:0.0:1786176786.637190:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176786.637197:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1786176786.637201:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1786176786.637204:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786176786.637208:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00080000:0.0:1786176786.637214:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11461 00000400:00000200:2.0:1786176786.637215:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176786.637223:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176786.637230:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426254:426254:256:4294967295] 192.168.202.132@tcp LPNI seq info [426216:426216:8:4294967295] 00000400:00000200:0.0:1786176786.637232:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176786.637239:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176786.637245:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176786.637251:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786176786.637262:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000040:0.0:1786176786.637268:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:2.0:1786176786.637273:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637279:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:2.0:1786176786.637282:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00000040:0.0:1786176786.637285:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637289:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:2.0:1786176786.637294:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:0.0:1786176786.637295:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:2.0:1786176786.637316:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000020:00000040:0.0:1786176786.637323:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000800:00000200:2.1:1786176786.637323:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:0.0:1786176786.637331:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:2.1:1786176786.637332:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000020:00000040:0.0:1786176786.637336:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176786.637345:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000100:2.0:1786176786.637346:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176786.637352:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000020:00000001:0.0:1786176786.637353:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176786.637385:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637388:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176786.637391:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637394:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176786.637397:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176786.637401:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637404:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176786.637407:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176786.637412:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176786.637415:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176786.637416:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1786176786.637418:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:2.0:1786176786.637424:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000008:00100000:0.0:1786176786.637426:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000200:2.0:1786176786.637432:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000008:00000001:0.0:1786176786.637434:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786176786.637435:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:0.0:1786176786.637441:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637441:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:0.0:1786176786.637449:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000100:00000200:2.0:1786176786.637450:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1786176786.637458:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176786.637470:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176786.637473:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637474:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1786176786.637480:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176786.637486:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1786176786.637486:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786176786.637493:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637502:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637504:0:2392:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:0.0:1786176786.637505:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176786.637510:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637511:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176786.637520:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786176786.637521:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176786.637530:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306137345 00000100:00000001:0.0:1786176786.637544:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000010:2.0:1786176786.637546:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c180. 00000100:00000001:0.0:1786176786.637548:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176786.637554:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.637555:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786176786.637566:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637567:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306137345 00000100:00080000:0.0:1786176786.637572:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:2.0:1786176786.637576:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c0c0. 00000100:00080000:0.0:1786176786.637578:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176786.637583:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1786176786.637584:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1786176786.637586:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176786.637589:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786176786.637590:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786176786.637592:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637597:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1786176786.637598:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11461 00000100:00100000:2.0:1786176786.637605:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176786/real 1786176786] req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1786176786.637614:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000001:2.0:1786176786.637623:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176786.637625:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1786176786.637630:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000010:2.0:1786176786.637630:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176786.637637:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000100:00080000:0.0:1786176786.637640:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000010:2.0:1786176786.637640:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176786.637650:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1786176786.637651:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637654:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176786.637659:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:2.0:1786176786.637661:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176786.637662:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176786.637668:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176786.637670:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176786.637675:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176786.637678:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176786.637684:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176786.637692:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176786.637699:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637703:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1786176786.637708:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786176786.637709:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637714:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176786.637723:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176786.637733:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176786.637741:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176786.637748:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786176786.637751:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:0.0:1786176786.637753:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176786.637755:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1786176786.637757:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637760:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786176786.637761:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176786.637764:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1786176786.637765:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176786.637771:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176786.637772:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176786.637775:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176786.637787:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000100:00000001:2.0:1786176786.637789:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:0.0:1786176786.637792:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176786.637798:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:2.0:1786176786.637799:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786176786.637804:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176786.637814:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176786.637818:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176786.637824:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637828:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:0.0:1786176786.637831:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:2.0:1786176786.637832:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.637839:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786176786.637840:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.637841:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.637848:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00100000:0.0:1786176786.637853:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176786.637862:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176786.637862:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786176786.637870:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176786.637878:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:0.0:1786176786.637879:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786176786.637885:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176786.637895:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176786.637912:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176786.637913:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176786.637921:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11471) 00000100:00100000:2.0:1786176786.637936:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235536000:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176786.637945:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.637948:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176786.637956:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09315c00 x1872930235536000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.637964:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176786.637966:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176786.637970:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786176786.637974:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176786.637976:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176786.637978:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786176786.637981:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176786.637984:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:2.0:1786176786.637988:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176786.637990:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176786.637992:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09315c00. 00000100:00000001:2.0:1786176786.637995:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176786.637997:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.637999:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.638007:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176786.638009:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.638012:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176786.638014:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176786.638016:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176786.638021:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638029:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176786.638031:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176786.638033:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176786.638036:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176786.638038:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176786.638042:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536128:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176786.638045:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176786.638047:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176786.638049:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176786.638051:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176786.638055:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786176786.638057:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176786.638062:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176786.638065:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176786.638068:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176786.638072:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235536128, portal 10 00000100:00000001:2.0:1786176786.638074:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176786.638076:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794212352 : -110371915339264 : ffff9b9e09316a00) 00000100:00000040:2.0:1786176786.638083:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638090:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176786.638093:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176786.638096:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176786.638099:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235536128, offset 0 00000400:00000200:2.0:1786176786.638103:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176786.638107:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176786.638112:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176786.638115:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176786.638118:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176786.638120:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176786.638123:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176786.638126:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426255:426255:256:4294967295] 192.168.202.132@tcp LPNI seq info [426217:426217:8:4294967295] 00000400:00000200:2.0:1786176786.638130:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176786.638136:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176786.638140:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176786.638144:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176786.638148:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176786.638151:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786176786.638156:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176786.638159:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786176786.638163:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176786.638165:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176786.638168:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176786.638172:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176786.638176:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176786.638178:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176786.638180:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176786.638185:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638198:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176786.638201:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638208:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.638210:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176786.638214:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176786.638218:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786176786.638222:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176786.638224:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306133508 00000100:00000001:2.0:1786176786.638227:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306137345 00000100:00000001:2.0:1786176786.638229:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306133508 00000100:00000001:2.0:1786176786.638232:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306137345 00000100:00000001:2.0:1786176786.638234:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176786.638236:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.638238:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176786.638243:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176786/real 1786176786] req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176786.638250:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176786.638252:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176786.638254:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176786.638257:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176786.638258:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176786.638260:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176786.638264:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176786.638272:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638278:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176786.638280:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.638282:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176786.638284:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176786.638288:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638294:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176786.638314:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638322:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176786.638325:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.638326:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176786.638328:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.638330:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176786.638332:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176786.638335:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176786.638338:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176786.638341:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176786.638345:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176786.638352:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536128:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176786.638356:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176786.638384:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638391:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176786.638392:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176786.638395:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176786.638397:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176786.638399:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176786.638402:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536256:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176786.638406:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176786.638408:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176786.638410:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176786.638412:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176786.638416:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176786.638418:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176786.638422:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176786.638424:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176786.638428:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176786.638431:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235536256, portal 10 00000100:00000001:2.0:1786176786.638433:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176786.638434:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215936 : -110371915335680 : ffff9b9e09317800) 00000100:00000040:2.0:1786176786.638440:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638447:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176786.638450:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176786.638452:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176786.638454:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235536256, offset 0 00000400:00000200:2.0:1786176786.638458:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176786.638462:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176786.638466:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176786.638469:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176786.638472:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176786.638474:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176786.638477:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176786.638480:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426256:426256:256:4294967295] 192.168.202.132@tcp LPNI seq info [426218:426218:8:4294967295] 00000400:00000200:2.0:1786176786.638485:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176786.638491:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176786.638495:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176786.638498:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176786.638502:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176786.638505:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786176786.638510:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176786.638512:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786176786.638516:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176786.638518:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176786.638521:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176786.638525:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176786.638528:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176786.638529:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176786.638531:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176786.638535:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638543:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176786.638546:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638552:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.638554:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176786.638557:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176786.638560:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786176786.638564:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176786.638566:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306133509 00000100:00000001:2.0:1786176786.638569:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306137345 00000100:00000001:2.0:1786176786.638571:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306133509 00000100:00000001:2.0:1786176786.638573:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306137345 00000100:00000001:2.0:1786176786.638575:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176786.638577:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.638579:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176786.638582:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176786/real 1786176786] req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176786.638590:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176786.638591:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176786.638593:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176786.638596:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176786.638597:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176786.638600:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176786.638603:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176786.638611:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638617:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176786.638618:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.638620:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176786.638621:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176786.638625:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638632:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176786.638637:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638644:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176786.638647:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.638648:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176786.638650:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.638652:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176786.638654:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176786.638656:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176786.638660:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176786.638662:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176786.638666:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176786.638680:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536256:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176786.638685:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176786.638689:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638697:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176786.638698:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176786.638701:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176786.638703:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176786.638705:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176786.638708:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536384:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176786.638712:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176786.638714:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176786.638716:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176786.638718:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176786.638721:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786176786.638724:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176786.638727:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176786.638730:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176786.638733:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176786.638736:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235536384, portal 4 00000100:00000001:2.0:1786176786.638739:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176786.638740:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794210560 : -110371915341056 : ffff9b9e09316300) 00000100:00000040:2.0:1786176786.638746:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638753:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176786.638755:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176786.638757:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176786.638760:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235536384, offset 0 00000400:00000200:2.0:1786176786.638763:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176786.638769:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176786.638773:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176786.638776:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176786.638778:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176786.638781:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176786.638783:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176786.638787:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426257:426257:256:4294967295] 192.168.202.132@tcp LPNI seq info [426219:426219:8:4294967295] 00000400:00000200:2.0:1786176786.638791:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176786.638798:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176786.638803:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176786.638806:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176786.638811:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176786.638814:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786176786.638818:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176786.638822:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786176786.638826:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176786.638828:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176786.638831:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176786.638836:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176786.638839:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176786.638841:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176786.638843:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176786.638848:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638856:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176786.638859:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638865:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.638866:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176786.638868:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176786.638871:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786176786.638873:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176786.638875:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306133509 00000100:00000001:2.0:1786176786.638877:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306137345 00000100:00000001:2.0:1786176786.638878:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306133509 00000100:00000001:2.0:1786176786.638880:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306137345 00000100:00000001:2.0:1786176786.638881:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176786.638882:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.638884:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176786.638886:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176786/real 1786176786] req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176786.638891:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176786.638892:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176786.638893:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176786.638894:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176786.638896:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176786.638897:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176786.638899:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176786.638905:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638908:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176786.638909:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.638910:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176786.638911:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176786.638913:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638917:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176786.638920:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638925:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176786.638927:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.638928:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176786.638929:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.638930:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176786.638931:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176786.638933:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176786.638935:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176786.638936:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176786.638939:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176786.638945:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536384:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176786.638949:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176786.638953:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.638960:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176786.638961:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176786.638963:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176786.638965:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176786.638968:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176786.638971:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536512:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176786.638974:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176786.638976:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176786.638978:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176786.638980:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176786.638983:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176786.638986:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176786.638989:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176786.638992:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176786.638995:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176786.638998:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235536512, portal 4 00000100:00000001:2.0:1786176786.639000:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176786.639001:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794202496 : -110371915349120 : ffff9b9e09314380) 00000100:00000040:2.0:1786176786.639006:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.639010:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176786.639012:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176786.639014:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176786.639015:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235536512, offset 0 00000400:00000200:2.0:1786176786.639018:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176786.639021:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176786.639023:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176786.639025:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176786.639027:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176786.639029:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176786.639030:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176786.639033:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426258:426258:256:4294967295] 192.168.202.132@tcp LPNI seq info [426220:426220:8:4294967295] 00000400:00000200:2.0:1786176786.639035:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176786.639040:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176786.639043:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176786.639045:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176786.639048:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176786.639049:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786176786.639052:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176786.639054:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786176786.639056:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176786.639058:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176786.639060:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176786.639062:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176786.639064:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176786.639065:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176786.639067:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176786.639069:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.639081:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176786.639085:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176811 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.639092:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.639093:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176786.639097:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176786.639100:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786176786.639103:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176786.639105:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306133511 00000100:00000001:2.0:1786176786.639108:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306137345 00000100:00000001:2.0:1786176786.639110:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306133511 00000100:00000001:2.0:1786176786.639112:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306137345 00000100:00000001:2.0:1786176786.639114:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176786.639116:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.639118:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176786.639121:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176786/real 1786176786] req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176786.639129:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176786.639130:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176786.639132:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176786.639135:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176786.639136:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176786.639139:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176786.639142:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176786.639151:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.639159:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176786.639161:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.639163:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176786.639164:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176786.639168:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.639175:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176786.639179:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.639187:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176786.639189:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.639191:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176786.639192:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.639194:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176786.639196:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176786.639199:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176786.639202:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176786.639205:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176786.639210:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176786.639218:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536512:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176786.639222:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.639224:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176786.639228:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316a00 x1872930235536128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.639235:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176786.639237:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176786.639239:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786176786.639242:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176786.639243:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176786.639246:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176786.639248:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176786.639250:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:2.0:1786176786.639252:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176786.639253:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176786.639255:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316a00. 00000100:00000001:2.0:1786176786.639258:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176786.639260:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.639262:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176786.639266:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317800 x1872930235536256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.639273:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176786.639274:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176786.639276:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176786.639278:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176786.639279:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176786.639281:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176786.639283:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176786.639284:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:2.0:1786176786.639287:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176786.639288:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176786.639289:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317800. 00000100:00000001:2.0:1786176786.639291:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176786.639293:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.639294:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176786.639312:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316300 x1872930235536384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.639318:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176786.639319:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176786.639321:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786176786.639322:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176786.639323:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176786.639324:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176786.639325:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176786.639326:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:2.0:1786176786.639328:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176786.639329:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176786.639330:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316300. 00000100:00000001:2.0:1786176786.639332:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176786.639333:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.639334:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176786.639338:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314380 x1872930235536512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176811 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176786.639345:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176786.639346:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176786.639348:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176786.639350:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176786.639352:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176786.639353:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176786.639355:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176786.639381:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:2.0:1786176786.639384:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176786.639385:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176786.639386:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314380. 00000100:00000001:2.0:1786176786.639390:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176786.639391:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.639393:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176786.639398:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176786.639399:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176786.639402:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176786.639404:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176787.148490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176787.148498:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176787.148501:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176787.148507:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176787.148512:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176787.148515:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176787.148519:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176787.148521:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176787.148524:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176787.532425:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176787.532431:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176787.532436:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176787.532437:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176787.532440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176787.532442:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176787.532501:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176787.532505:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176787.532507:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176788.172478:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176788.172487:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176788.172489:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176788.172494:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176788.172498:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176788.172502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176788.172504:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176788.172506:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176788.172510:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176788.556469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176788.556474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176788.556482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176788.556487:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176788.556490:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176788.556494:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176788.556498:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176788.556501:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176788.556504:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176788.556508:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176788.556511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176788.556512:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176788.556597:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176788.556604:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176788.556609:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176789.580425:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176789.580431:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176789.580438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176789.580450:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176789.580455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176789.580458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176789.580558:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176789.580566:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176789.580571:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176790.540454:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306137345, at 4306137408 00000100:00000010:0.0:1786176790.540469:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c180. 00010000:00000001:0.0:1786176790.604456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176790.604461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176790.604468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176790.604471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176790.604474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176790.604476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176790.604597:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176790.604603:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176790.604607:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176791.244552:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176791.244559:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176791.244560:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176791.244565:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176791.244567:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176791.244570:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.244573:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176791.244575:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.244577:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176791.628469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176791.628481:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176791.628489:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176791.628495:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176791.628504:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176791.628506:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176791.628511:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176791.628518:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176791.628529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176791.628532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176791.628537:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176791.628539:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176791.628540:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176791.628542:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176791.628543:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176791.628545:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176791.628546:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176791.628547:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176791.628615:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176791.628619:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176791.628622:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176791.756811:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176791.756816:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176791.756819:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176791.756829:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb480 x1872930235536640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.756846:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176791.756849:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176791.756852:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176791.756856:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176791.756860:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176791.756864:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235536640:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176791.756867:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176791.756871:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176791.756875:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176791.756880:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176791.756887:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786176791.756891:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176791.756899:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176791.756903:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176791.756909:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176791.756917:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235536640, portal 25 00000100:00000001:2.0:1786176791.756921:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176791.756924:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487995520 : -110371221556096 : ffff9b9e328bb480) 00000100:00000040:2.0:1786176791.756932:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb480 x1872930235536640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.756942:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176791.756948:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176791.756955:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176791.756959:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235536640, offset 0 00000400:00000200:2.0:1786176791.756969:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176791.756980:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176791.756988:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176791.756993:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176791.756999:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176791.757005:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176791.757011:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176791.757018:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426259:426259:256:4294967295] 192.168.202.132@tcp LPNI seq info [426221:426221:8:4294967295] 00000400:00000200:2.0:1786176791.757025:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176791.757042:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176791.757052:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176791.757058:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176791.757067:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176791.757073:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786176791.757084:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176791.757088:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1786176791.757091:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176791.757093:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176791.757098:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176791.757105:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176791.757114:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176791.757118:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176791.757123:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176791.757129:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb480 x1872930235536640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.757148:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176791.757154:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb480 x1872930235536640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.757173:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.757177:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176791.757184:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176791.757191:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176791.757196:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306142464 00000100:00000010:2.0:1786176791.757204:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c0c0. 00000100:00000001:2.0:1786176791.757212:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786176791.757220:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306142464 00000100:00000010:2.0:1786176791.757224:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c180. 00000100:00000001:2.0:1786176791.757229:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786176791.757233:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176791.757237:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.757239:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176791.757246:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176791/real 1786176791] req@ffff9b9e328bb480 x1872930235536640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176791.757271:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176791.757274:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176791.757278:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176791.757283:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176791.757286:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176791.757289:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176791.757293:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb480 x1872930235536640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176791.757303:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb480 x1872930235536640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.757311:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176791.757313:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.757315:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176791.757316:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176791.757323:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb480 x1872930235536640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.757347:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176791.757388:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb480 x1872930235536640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.757398:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176791.757406:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.757411:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176791.757415:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.757421:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176791.757425:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176791.757432:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176791.757441:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176791.757447:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176791.757457:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb480 x1872930235536640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176791.757471:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235536640:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176791.757482:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.757485:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176791.757490:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb480 x1872930235536640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.757503:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176791.757507:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176791.757511:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786176791.757516:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176791.757520:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176791.757524:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786176791.757530:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176791.757535:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:2.0:1786176791.757543:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176791.757548:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176791.757552:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb480. 00000100:00000001:2.0:1786176791.757559:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176791.757564:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.757570:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.757580:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176791.757583:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.757587:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176791.757591:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176791.757594:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176791.757600:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9f80 x1872930235536768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.757614:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176791.757619:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176791.757624:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176791.757629:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176791.757637:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176791.757646:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536768:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176791.757654:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176791.757658:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176791.757663:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176791.757668:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176791.757673:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786176791.757677:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176791.757695:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176791.757702:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176791.757707:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176791.757712:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235536768, portal 10 00000100:00000001:2.0:1786176791.757717:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176791.757720:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487990144 : -110371221561472 : ffff9b9e328b9f80) 00000100:00000040:2.0:1786176791.757732:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9f80 x1872930235536768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.757742:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176791.757748:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176791.757752:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176791.757755:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235536768, offset 0 00000400:00000200:2.0:1786176791.757761:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176791.757769:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176791.757778:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176791.757784:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176791.757788:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176791.757794:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176791.757801:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176791.757806:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426260:426260:256:4294967295] 192.168.202.132@tcp LPNI seq info [426222:426222:8:4294967295] 00000400:00000200:2.0:1786176791.757811:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176791.757823:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176791.757832:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176791.757838:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176791.757845:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176791.757850:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786176791.757859:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176791.757862:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1786176791.757871:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176791.757874:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176791.757880:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176791.757885:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176791.757890:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176791.757893:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176791.757898:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176791.757904:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9f80 x1872930235536768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.757921:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176791.757927:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235536768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.757941:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.757944:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176791.757948:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176791.757954:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786176791.757960:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176791.757964:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306138628 00000100:00000001:2.0:1786176791.757970:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306142464 00000100:00000001:2.0:1786176791.757974:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306138628 00000100:00000001:2.0:1786176791.757979:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306142464 00000100:00000001:2.0:1786176791.757982:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176791.757986:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.757990:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176791.757997:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176791/real 1786176791] req@ffff9b9e328b9f80 x1872930235536768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176791.758025:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176791.758028:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176791.758029:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176791.758032:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176791.758033:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176791.758036:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176791.758039:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9f80 x1872930235536768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176791.758045:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9f80 x1872930235536768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758049:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176791.758050:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758051:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176791.758052:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176791.758054:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9f80 x1872930235536768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758058:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176791.758061:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9f80 x1872930235536768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758065:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176791.758066:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758067:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176791.758068:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758069:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176791.758071:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176791.758072:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176791.758074:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176791.758076:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176791.758079:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9f80 x1872930235536768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176791.758085:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536768:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176791.758087:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176791.758089:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb800 x1872930235536896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758093:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176791.758094:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176791.758096:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176791.758097:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176791.758098:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176791.758100:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536896:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176791.758102:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176791.758103:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176791.758104:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176791.758105:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176791.758108:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176791.758109:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176791.758111:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176791.758113:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176791.758115:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176791.758117:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235536896, portal 10 00000100:00000001:2.0:1786176791.758118:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176791.758119:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487996416 : -110371221555200 : ffff9b9e328bb800) 00000100:00000040:2.0:1786176791.758123:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb800 x1872930235536896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758128:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176791.758129:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176791.758130:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176791.758132:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235536896, offset 0 00000400:00000200:2.0:1786176791.758135:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176791.758137:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176791.758140:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176791.758142:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176791.758144:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176791.758146:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176791.758147:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176791.758149:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426261:426261:256:4294967295] 192.168.202.132@tcp LPNI seq info [426223:426223:8:4294967295] 00000400:00000200:2.0:1786176791.758153:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176791.758157:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176791.758159:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176791.758161:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176791.758164:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176791.758166:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786176791.758169:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176791.758171:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1786176791.758173:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176791.758174:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176791.758177:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176791.758179:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176791.758181:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176791.758182:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176791.758184:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176791.758186:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb800 x1872930235536896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758191:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176791.758193:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb800 x1872930235536896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758197:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.758198:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176791.758200:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176791.758202:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786176791.758203:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176791.758205:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306138628 00000100:00000001:2.0:1786176791.758207:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306142464 00000100:00000001:2.0:1786176791.758209:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306138628 00000100:00000001:2.0:1786176791.758210:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306142464 00000100:00000001:2.0:1786176791.758211:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176791.758212:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.758213:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176791.758215:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176791/real 1786176791] req@ffff9b9e328bb800 x1872930235536896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176791.758220:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176791.758221:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176791.758222:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176791.758223:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176791.758224:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176791.758225:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176791.758228:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb800 x1872930235536896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176791.758233:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb800 x1872930235536896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758237:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176791.758237:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758238:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176791.758239:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176791.758241:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb800 x1872930235536896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758245:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176791.758247:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb800 x1872930235536896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758270:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176791.758273:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758274:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176791.758275:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758276:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176791.758277:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176791.758279:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176791.758281:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176791.758283:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176791.758285:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb800 x1872930235536896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176791.758291:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235536896:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176791.758293:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758294:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176791.758296:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9f80 x1872930235536768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758300:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176791.758301:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176791.758303:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786176791.758304:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176791.758305:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176791.758306:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176791.758308:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176791.758309:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:2.0:1786176791.758311:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176791.758312:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176791.758313:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9f80. 00000100:00000001:2.0:1786176791.758316:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176791.758317:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758319:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176791.758322:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb800 x1872930235536896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758336:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176791.758337:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176791.758339:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176791.758342:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176791.758343:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176791.758344:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176791.758346:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176791.758348:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261adc00. 02000000:00000001:2.0:1786176791.758350:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176791.758352:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176791.758353:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb800. 00000100:00000001:2.0:1786176791.758355:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176791.758374:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758376:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758378:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176791.758379:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.758381:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176791.758382:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176791.758383:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176791.758386:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9180 x1872930235537024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758391:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176791.758392:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176791.758394:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176791.758395:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176791.758397:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176791.758400:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537024:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176791.758404:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176791.758406:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176791.758407:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176791.758409:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176791.758413:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176791.758415:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176791.758419:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176791.758422:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176791.758426:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176791.758430:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235537024, portal 4 00000100:00000001:2.0:1786176791.758432:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176791.758434:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487986560 : -110371221565056 : ffff9b9e328b9180) 00000100:00000040:2.0:1786176791.758440:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9180 x1872930235537024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758446:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176791.758449:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176791.758451:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176791.758454:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235537024, offset 0 00000400:00000200:2.0:1786176791.758458:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176791.758462:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176791.758466:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176791.758469:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176791.758472:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176791.758474:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176791.758475:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176791.758478:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426262:426262:256:4294967295] 192.168.202.132@tcp LPNI seq info [426224:426224:8:4294967295] 00000400:00000200:2.0:1786176791.758481:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176791.758486:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176791.758489:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176791.758491:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176791.758494:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176791.758496:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786176791.758498:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176791.758500:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1786176791.758503:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176791.758504:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176791.758506:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176791.758509:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176791.758511:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176791.758513:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176791.758514:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176791.758517:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9180 x1872930235537024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758525:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176791.758528:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9180 x1872930235537024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758535:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.758537:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176791.758540:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176791.758543:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786176791.758546:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176791.758548:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306138629 00000100:00000001:2.0:1786176791.758551:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306142464 00000100:00000001:2.0:1786176791.758553:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306138629 00000100:00000001:2.0:1786176791.758556:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306142464 00000100:00000001:2.0:1786176791.758558:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176791.758560:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.758562:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176791.758566:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176791/real 1786176791] req@ffff9b9e328b9180 x1872930235537024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176791.758575:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176791.758576:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176791.758578:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176791.758580:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176791.758581:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176791.758583:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176791.758585:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9180 x1872930235537024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176791.758591:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9180 x1872930235537024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758596:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176791.758597:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758598:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176791.758599:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176791.758601:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9180 x1872930235537024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758605:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176791.758608:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9180 x1872930235537024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758616:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176791.758618:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758620:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176791.758622:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758624:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176791.758625:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176791.758628:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176791.758632:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176791.758634:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176791.758639:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9180 x1872930235537024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176791.758647:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537024:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176791.758651:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176791.758655:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8a80 x1872930235537152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758662:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176791.758663:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176791.758667:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176791.758669:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176791.758671:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176791.758674:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537152:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176791.758677:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176791.758679:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176791.758681:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176791.758683:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176791.758685:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786176791.758687:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176791.758689:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176791.758691:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176791.758693:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176791.758695:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235537152, portal 4 00000100:00000001:2.0:1786176791.758697:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176791.758698:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487984768 : -110371221566848 : ffff9b9e328b8a80) 00000100:00000040:2.0:1786176791.758703:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8a80 x1872930235537152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758707:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176791.758708:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176791.758710:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176791.758711:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235537152, offset 0 00000400:00000200:2.0:1786176791.758714:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176791.758717:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176791.758719:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176791.758721:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176791.758723:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176791.758724:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176791.758726:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176791.758728:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426263:426263:256:4294967295] 192.168.202.132@tcp LPNI seq info [426225:426225:8:4294967295] 00000400:00000200:2.0:1786176791.758731:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176791.758735:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176791.758738:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176791.758741:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176791.758743:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176791.758745:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786176791.758748:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176791.758750:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1786176791.758752:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176791.758754:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176791.758756:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176791.758759:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176791.758761:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176791.758762:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176791.758764:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176791.758766:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8a80 x1872930235537152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758771:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176791.758773:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8a80 x1872930235537152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758777:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.758778:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176791.758779:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176791.758781:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786176791.758783:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176791.758785:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306138631 00000100:00000001:2.0:1786176791.758786:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306142464 00000100:00000001:2.0:1786176791.758788:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306138631 00000100:00000001:2.0:1786176791.758789:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306142464 00000100:00000001:2.0:1786176791.758790:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176791.758792:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.758793:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176791.758795:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176791/real 1786176791] req@ffff9b9e328b8a80 x1872930235537152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176791.758801:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176791.758802:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176791.758803:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176791.758804:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176791.758805:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176791.758806:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176791.758808:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8a80 x1872930235537152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176791.758813:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8a80 x1872930235537152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758818:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176791.758819:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758820:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176791.758821:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176791.758823:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8a80 x1872930235537152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758827:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176791.758829:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8a80 x1872930235537152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758833:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176791.758834:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758835:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176791.758836:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758837:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176791.758838:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176791.758840:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176791.758841:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176791.758843:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176791.758846:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8a80 x1872930235537152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176791.758851:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537152:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176791.758853:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758854:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176791.758856:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9180 x1872930235537024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758860:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176791.758861:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176791.758862:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176791.758863:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176791.758864:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176791.758865:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176791.758866:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176791.758867:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:2.0:1786176791.758869:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176791.758870:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176791.758871:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9180. 00000100:00000001:2.0:1786176791.758872:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176791.758873:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758874:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176791.758876:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8a80 x1872930235537152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176791.758880:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176791.758881:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176791.758882:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786176791.758883:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176791.758883:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176791.758884:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176791.758885:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176791.758886:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af000. 02000000:00000001:2.0:1786176791.758887:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176791.758889:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176791.758890:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8a80. 00000100:00000001:2.0:1786176791.758891:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176791.758892:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758893:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176791.758896:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176791.758897:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176791.758898:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176791.758899:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176792.268463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176792.268470:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176792.268472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176792.268477:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176792.268481:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176792.268484:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176792.268486:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176792.268488:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176792.268490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176792.652468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176792.652474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176792.652483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176792.652487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176792.652495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176792.652498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176792.652565:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176792.652568:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176792.652570:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176793.292459:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176793.292465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176793.292467:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176793.292472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176793.292475:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176793.292478:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176793.292480:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176793.292482:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176793.292485:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176793.676442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176793.676445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176793.676450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176793.676451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176793.676454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176793.676455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176793.676534:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176793.676538:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176793.676540:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176794.316490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176794.316498:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176794.316500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176794.316505:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176794.316508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176794.316511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176794.316514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176794.316517:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176794.316520:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176794.700456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176794.700461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176794.700469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176794.700472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176794.700477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176794.700480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176794.700527:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176794.700530:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176794.700533:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176795.340454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176795.340462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176795.340465:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176795.340472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176795.340477:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176795.340481:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176795.340484:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176795.340487:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176795.340490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176795.660430:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306142464, at 4306142528 00000100:00000010:0.0:1786176795.660444:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c0c0. 00010000:00000001:0.0:1786176795.724447:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176795.724455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176795.724461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176795.724464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176795.724471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176795.724473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176795.724533:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176795.724537:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176795.724540:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176796.364462:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176796.364468:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176796.364470:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176796.364474:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176796.364478:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176796.364482:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.364485:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176796.364487:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.364490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176796.748432:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176796.748436:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176796.748440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176796.748443:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176796.748448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176796.748451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176796.748508:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176796.748511:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176796.748515:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176796.876458:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176796.876470:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176796.876473:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176796.876477:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176796.876482:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176796.876490:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11471 00000400:00000200:0.0:1786176796.876500:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176796.876508:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176796.876511:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176796.876515:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176796.876519:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176796.876522:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176796.876525:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176796.876530:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176796.876532:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176796.876535:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176796.876539:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176796.876542:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176796.876545:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176796.876549:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176796.876551:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176796.876554:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176796.876556:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176796.876559:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176796.876563:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176796.876566:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176796.876569:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176796.876572:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176796.876576:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176796.876578:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176796.876581:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176796.876592:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786176796.876596:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176796.876600:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176796.876601:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176796.876605:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176796.876608:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176796.876612:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176796.876641:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176796.876649:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176796.876660:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176796.876671:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176796.876675:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176796.876681:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176796.876705:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176796.876709:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176796.876713:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176796.876716:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176796.876718:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176796.876722:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176796.876726:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11471 00000100:00000001:2.0:1786176796.876729:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1786176796.876731:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176796.876735:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1786176796.876737:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176796.876741:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786176796.876741:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786176796.876747:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176796.876752:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176796.876753:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786176796.876756:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176796.876762:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176796.876768:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176796.876771:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000100:00000001:2.0:1786176796.876773:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1786176796.876775:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1786176796.876778:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1786176796.876779:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176796.876784:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 02000000:00000001:2.0:1786176796.876785:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1786176796.876787:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 02000000:00000001:2.0:1786176796.876790:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1786176796.876794:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:2.0:1786176796.876796:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176796.876803:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1786176796.876805:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235537280:192.168.202.132@tcp:250:kworker.0 00000100:00000040:0.0:1786176796.876809:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176796.876811:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1786176796.876815:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:2.0:1786176796.876816:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:0.0:1786176796.876820:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 02000000:00000001:2.0:1786176796.876821:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176796.876825:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1786176796.876828:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000010:2.0:1786176796.876831:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:0.0:1786176796.876833:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176796.876835:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176796.876837:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1786176796.876843:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:0.0:1786176796.876844:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1786176796.876847:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 02000000:00000001:0.0:1786176796.876849:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176796.876853:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:2.0:1786176796.876854:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000008:00100000:0.0:1786176796.876857:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000200:2.0:1786176796.876861:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235537280, portal 25 00000008:00000001:0.0:1786176796.876862:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.876867:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786176796.876868:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.876870:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794202496 : -110371915349120 : ffff9b9e09314380) 00000100:00000010:0.0:1786176796.876876:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000100:00000040:2.0:1786176796.876877:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1786176796.876883:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000100:00000001:2.0:1786176796.876886:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1786176796.876890:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176796.876892:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176796.876895:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176796.876896:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176796.876899:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176796.876899:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176796.876906:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 02000000:00000010:0.0:1786176796.876908:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176796.876913:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786176796.876913:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235537280, offset 0 00000100:00100000:0.0:1786176796.876924:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786176796.876928:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:0.0:1786176796.876938:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1786176796.876942:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786176796.876943:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176796.876952:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786176796.876953:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176796.876963:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1786176796.876970:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176796.876971:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00080000:0.0:1786176796.876977:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1786176796.876979:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1786176796.876985:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:2.0:1786176796.876986:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:0.0:1786176796.876988:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176796.876992:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1786176796.876995:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426264:426264:256:4294967295] 192.168.202.132@tcp LPNI seq info [426226:426226:8:4294967295] 00000100:00000001:0.0:1786176796.876997:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786176796.877004:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:0.0:1786176796.877005:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11471 00000400:00000200:2.0:1786176796.877015:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000400:00000200:0.0:1786176796.877024:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:2.0:1786176796.877024:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176796.877032:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000400:00000200:0.0:1786176796.877033:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176796.877038:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:2.0:1786176796.877042:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:0.0:1786176796.877045:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:2.0:1786176796.877047:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:0.0:1786176796.877050:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176796.877056:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:2.1:1786176796.877057:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786176796.877061:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:2.1:1786176796.877062:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000020:00000001:0.0:1786176796.877066:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000100:2.0:1786176796.877070:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000020:00000040:0.0:1786176796.877071:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000800:00000010:2.0:1786176796.877074:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000020:00000001:0.0:1786176796.877075:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176796.877079:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:2.0:1786176796.877081:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:0.0:1786176796.877082:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176796.877087:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786176796.877088:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1786176796.877096:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:2.0:1786176796.877097:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176796.877100:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176796.877105:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176796.877109:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176796.877111:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176796.877118:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176796.877123:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176796.877130:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00000001:2.0:1786176796.877130:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1786176796.877135:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000040:2.0:1786176796.877136:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176796.877141:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176796.877146:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786176796.877149:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176796.877151:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176796.877151:0:2392:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1786176796.877157:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.877158:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176796.877160:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176796.877164:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1786176796.877164:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176796.877168:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306147584 00000008:00000001:0.0:1786176796.877170:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176796.877175:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176796.877175:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cf60. 00000100:00000010:0.0:1786176796.877183:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000100:00000001:2.0:1786176796.877184:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000020:00000040:0.0:1786176796.877189:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000001:2.0:1786176796.877192:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306147584 02000000:00000001:0.0:1786176796.877194:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000010:2.0:1786176796.877197:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cd20. 02000000:00000001:0.0:1786176796.877198:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176796.877215:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1786176796.877218:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1786176796.877219:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176796.877223:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.877224:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176796.877229:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.877231:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000010:0.0:1786176796.877232:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00100000:2.0:1786176796.877236:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176796/real 1786176796] req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176796.877239:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1786176796.877250:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:0.0:1786176796.877252:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1786176796.877252:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176796.877256:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176796.877259:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176796.877262:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176796.877264:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176796.877268:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176796.877283:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176796.877285:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176796.877291:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.877293:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176796.877296:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.877299:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1786176796.877301:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.877302:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176796.877308:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.877333:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176796.877338:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176796.877339:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1786176796.877344:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786176796.877347:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786176796.877350:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786176796.877351:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176796.877353:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1786176796.877354:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:0.0:1786176796.877356:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176796.877383:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.877387:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176796.877389:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176796.877393:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176796.877398:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176796.877402:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1786176796.877408:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:2.0:1786176796.877412:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1786176796.877416:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11471 00000400:00000200:0.0:1786176796.877425:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176796.877430:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176796.877434:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1786176796.877437:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235537280:192.168.202.132@tcp:250:kworker.0 00000100:00000040:0.0:1786176796.877438:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176796.877443:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176796.877446:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176796.877448:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176796.877449:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786176796.877452:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1786176796.877453:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176796.877456:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000040:2.0:1786176796.877457:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314380 x1872930235537280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1786176796.877458:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176796.877462:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176796.877467:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176796.877472:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:2.0:1786176796.877473:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:0.0:1786176796.877477:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:2.0:1786176796.877478:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1786176796.877481:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:2.0:1786176796.877482:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 00000100:00000040:0.0:1786176796.877485:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:2.0:1786176796.877487:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1786176796.877488:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176796.877491:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786176796.877492:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000020:00000040:2.0:1786176796.877495:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00080000:0.0:1786176796.877497:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000020:00000001:2.0:1786176796.877498:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:0.0:1786176796.877500:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176796.877502:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786176796.877503:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176796.877506:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786176796.877507:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176796.877509:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:0.0:1786176796.877512:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176796.877513:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314380. 00000100:00000001:0.0:1786176796.877516:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176796.877518:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000008:00100000:0.0:1786176796.877519:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1786176796.877520:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1786176796.877523:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.877524:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176796.877527:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176796.877532:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000100:00000001:2.0:1786176796.877533:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:0.0:1786176796.877535:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:2.0:1786176796.877536:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176796.877540:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786176796.877541:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1786176796.877543:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.877544:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176796.877547:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1786176796.877547:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1786176796.877550:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1786176796.877554:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786176796.877555:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176796.877561:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:2.0:1786176796.877565:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176796.877566:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.877569:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176796.877574:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1786176796.877574:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176796.877579:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786176796.877586:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1786176796.877586:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176796.877590:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176796.877595:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537408:192.168.202.132@tcp:38:kworker.0 00000100:00000040:0.0:1786176796.877597:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.877606:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176796.877611:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1786176796.877612:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176796.877616:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1786176796.877616:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176796.877622:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176796.877623:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176796.877624:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176796.877626:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176796.877627:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11471 00000400:00000200:0.0:1786176796.877629:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176796.877631:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176796.877632:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1786176796.877634:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1786176796.877635:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176796.877638:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176796.877640:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000010:2.0:1786176796.877641:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:0.0:1786176796.877642:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176796.877647:0:191280:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:2.0:1786176796.877648:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1786176796.877649:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176796.877652:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176796.877654:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176796.877656:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000200:2.0:1786176796.877659:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:0.0:1786176796.877660:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176796.877665:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:2.0:1786176796.877665:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00000001:0.0:1786176796.877671:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176796.877673:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:2.0:1786176796.877673:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000001:0.0:1786176796.877678:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176796.877680:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000200:2.0:1786176796.877680:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235537408, portal 10 00000100:00000001:2.0:1786176796.877685:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1786176796.877686:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176796.877688:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794210560 : -110371915341056 : ffff9b9e09316300) 00000100:00000001:0.0:1786176796.877691:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176796.877694:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176796.877697:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1786176796.877698:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786176796.877701:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176796.877705:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176796.877708:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176796.877714:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.877715:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1786176796.877717:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176796.877720:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000010:0.0:1786176796.877723:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000400:00000010:2.0:1786176796.877724:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000020:00000040:0.0:1786176796.877727:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000200:2.0:1786176796.877728:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235537408, offset 0 02000000:00000001:0.0:1786176796.877731:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176796.877735:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176796.877735:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:0.0:1786176796.877738:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176796.877741:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1786176796.877743:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:0.0:1786176796.877744:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176796.877749:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 02000000:00000010:0.0:1786176796.877750:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000400:00000200:2.0:1786176796.877754:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1786176796.877755:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176796.877759:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00100000:0.0:1786176796.877763:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786176796.877764:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176796.877769:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176796.877774:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426265:426265:256:4294967295] 192.168.202.132@tcp LPNI seq info [426227:426227:8:4294967295] 00000100:00000001:0.0:1786176796.877782:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1786176796.877784:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786176796.877787:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176796.877793:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786176796.877797:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176796.877807:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:0.0:1786176796.877810:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1786176796.877812:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00000040:0.0:1786176796.877815:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11481) 00000800:00000200:2.0:1786176796.877819:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176796.877822:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176796.877825:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176796.877828:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786176796.877831:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176796.877832:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176796.877834:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176796.877837:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176796.877839:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176796.877840:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176796.877842:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176796.877844:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.877850:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176796.877852:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.877860:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.877862:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176796.877864:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176796.877867:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786176796.877869:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176796.877871:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306143748 00000100:00000001:2.0:1786176796.877873:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306147584 00000100:00000001:2.0:1786176796.877875:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306143748 00000100:00000001:2.0:1786176796.877876:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306147584 00000100:00000001:2.0:1786176796.877877:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176796.877879:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.877880:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176796.877882:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176796/real 1786176796] req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176796.877887:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176796.877888:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176796.877889:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176796.877891:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176796.877892:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176796.877893:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176796.877895:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176796.877900:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.877904:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176796.877905:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.877906:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176796.877907:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176796.877910:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.877915:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176796.877917:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.877922:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176796.877924:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.877924:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176796.877925:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.877926:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176796.877928:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176796.877930:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176796.877931:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176796.877933:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176796.877935:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176796.877940:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537408:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176796.877942:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176796.877944:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.877949:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176796.877950:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176796.877951:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176796.877953:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176796.877954:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176796.877956:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537536:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176796.877957:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176796.877959:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176796.877961:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176796.877962:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176796.877964:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176796.877966:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176796.877968:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176796.877970:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176796.877971:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176796.877973:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235537536, portal 10 00000100:00000001:2.0:1786176796.877974:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176796.877975:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215936 : -110371915335680 : ffff9b9e09317800) 00000100:00000040:2.0:1786176796.877979:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.877984:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176796.877985:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176796.877986:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176796.877988:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235537536, offset 0 00000400:00000200:2.0:1786176796.877991:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176796.877993:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176796.877995:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176796.877997:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176796.877998:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176796.878000:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176796.878001:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176796.878003:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426266:426266:256:4294967295] 192.168.202.132@tcp LPNI seq info [426228:426228:8:4294967295] 00000400:00000200:2.0:1786176796.878006:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176796.878011:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176796.878014:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176796.878015:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176796.878018:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176796.878020:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176796.878022:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176796.878024:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786176796.878027:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176796.878029:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176796.878031:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176796.878033:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176796.878035:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176796.878036:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176796.878038:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176796.878040:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878045:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176796.878047:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878051:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.878052:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176796.878054:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176796.878056:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786176796.878058:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176796.878059:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306143748 00000100:00000001:2.0:1786176796.878060:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306147584 00000100:00000001:2.0:1786176796.878062:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306143748 00000100:00000001:2.0:1786176796.878063:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306147584 00000100:00000001:2.0:1786176796.878065:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176796.878066:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.878067:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176796.878069:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176796/real 1786176796] req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176796.878073:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176796.878074:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176796.878075:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176796.878076:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176796.878077:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176796.878079:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176796.878081:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176796.878087:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878090:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176796.878091:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878092:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176796.878093:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176796.878095:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878099:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176796.878101:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878105:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176796.878107:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878108:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176796.878108:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878109:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176796.878111:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176796.878112:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176796.878114:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176796.878115:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176796.878117:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176796.878122:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537536:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176796.878124:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878126:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176796.878128:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316300 x1872930235537408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878132:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176796.878133:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176796.878134:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786176796.878136:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176796.878137:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176796.878137:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176796.878139:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176796.878140:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:2.0:1786176796.878141:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176796.878142:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176796.878143:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316300. 00000100:00000001:2.0:1786176796.878145:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176796.878146:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878147:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176796.878149:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317800 x1872930235537536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878153:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176796.878154:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176796.878155:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176796.878156:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176796.878157:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176796.878158:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176796.878159:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176796.878159:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:2.0:1786176796.878161:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176796.878161:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176796.878162:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317800. 00000100:00000001:2.0:1786176796.878163:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176796.878164:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878165:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878168:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176796.878169:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.878170:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176796.878171:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176796.878172:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176796.878174:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878179:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176796.878179:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176796.878181:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176796.878182:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176796.878183:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176796.878185:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537664:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176796.878187:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176796.878188:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176796.878189:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176796.878190:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176796.878192:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176796.878193:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176796.878196:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176796.878197:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176796.878213:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176796.878215:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235537664, portal 4 00000100:00000001:2.0:1786176796.878217:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176796.878218:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794212352 : -110371915339264 : ffff9b9e09316a00) 00000100:00000040:2.0:1786176796.878222:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878226:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176796.878228:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176796.878229:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176796.878231:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235537664, offset 0 00000400:00000200:2.0:1786176796.878233:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176796.878235:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176796.878238:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176796.878240:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176796.878241:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176796.878243:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176796.878245:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176796.878247:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426267:426267:256:4294967295] 192.168.202.132@tcp LPNI seq info [426229:426229:8:4294967295] 00000400:00000200:2.0:1786176796.878250:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176796.878254:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176796.878256:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176796.878258:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176796.878261:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176796.878263:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176796.878265:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176796.878267:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786176796.878270:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176796.878271:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176796.878273:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176796.878275:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176796.878277:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176796.878278:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176796.878280:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176796.878282:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878287:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176796.878289:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878293:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.878294:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176796.878296:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176796.878298:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786176796.878300:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176796.878301:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306143748 00000100:00000001:2.0:1786176796.878303:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306147584 00000100:00000001:2.0:1786176796.878304:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306143748 00000100:00000001:2.0:1786176796.878307:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306147584 00000100:00000001:2.0:1786176796.878308:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176796.878309:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.878311:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176796.878313:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176796/real 1786176796] req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176796.878318:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176796.878319:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176796.878320:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176796.878321:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176796.878322:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176796.878324:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176796.878326:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176796.878331:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878335:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176796.878336:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878337:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176796.878338:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176796.878340:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878344:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176796.878346:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878350:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176796.878352:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878353:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176796.878353:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878355:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176796.878370:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176796.878373:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176796.878375:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176796.878377:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176796.878379:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176796.878385:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537664:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176796.878387:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176796.878389:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878394:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176796.878395:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176796.878396:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176796.878398:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176796.878399:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176796.878400:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537792:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176796.878402:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176796.878403:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176796.878404:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176796.878406:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176796.878408:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786176796.878409:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176796.878411:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176796.878413:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176796.878415:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176796.878417:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235537792, portal 4 00000100:00000001:2.0:1786176796.878418:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176796.878419:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794208768 : -110371915342848 : ffff9b9e09315c00) 00000100:00000040:2.0:1786176796.878423:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878428:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176796.878429:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176796.878431:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176796.878432:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235537792, offset 0 00000400:00000200:2.0:1786176796.878435:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176796.878437:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176796.878439:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176796.878441:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176796.878442:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176796.878444:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176796.878445:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176796.878448:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426268:426268:256:4294967295] 192.168.202.132@tcp LPNI seq info [426230:426230:8:4294967295] 00000400:00000200:2.0:1786176796.878450:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176796.878455:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176796.878458:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176796.878460:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176796.878463:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176796.878465:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786176796.878467:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176796.878469:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786176796.878472:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176796.878473:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176796.878475:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176796.878478:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176796.878480:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176796.878481:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176796.878483:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176796.878485:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878490:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176796.878492:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176821 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878501:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.878502:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176796.878504:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176796.878506:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786176796.878508:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176796.878509:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306143751 00000100:00000001:2.0:1786176796.878511:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306147584 00000100:00000001:2.0:1786176796.878512:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306143751 00000100:00000001:2.0:1786176796.878513:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306147584 00000100:00000001:2.0:1786176796.878515:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176796.878516:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.878517:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176796.878519:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176796/real 1786176796] req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176796.878523:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176796.878524:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176796.878525:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176796.878527:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176796.878528:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176796.878529:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176796.878531:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176796.878536:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878539:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176796.878540:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878541:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176796.878542:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176796.878544:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878549:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176796.878551:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878555:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176796.878557:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878558:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176796.878559:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878560:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176796.878561:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176796.878562:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176796.878564:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176796.878565:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176796.878567:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176796.878572:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235537792:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176796.878574:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878575:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176796.878577:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316a00 x1872930235537664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878580:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176796.878581:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176796.878583:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176796.878584:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176796.878585:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176796.878586:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176796.878587:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176796.878588:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:2.0:1786176796.878590:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176796.878591:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176796.878592:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316a00. 00000100:00000001:2.0:1786176796.878594:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176796.878595:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878596:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176796.878598:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09315c00 x1872930235537792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176821 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176796.878602:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176796.878602:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176796.878603:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786176796.878604:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176796.878605:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176796.878606:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176796.878607:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176796.878607:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:2.0:1786176796.878609:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176796.878609:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176796.878610:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09315c00. 00000100:00000001:2.0:1786176796.878611:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176796.878612:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878613:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176796.878615:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176796.878616:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176796.878617:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176796.878618:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176797.388449:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176797.388456:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176797.388458:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176797.388462:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176797.388465:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176797.388470:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176797.388473:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176797.388476:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176797.388480:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176797.772454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176797.772460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176797.772468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176797.772471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176797.772479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176797.772483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176797.772557:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176797.772562:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176797.772566:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176798.412477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176798.412486:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176798.412489:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176798.412495:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176798.412500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176798.412504:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176798.412507:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176798.412510:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176798.412514:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176798.796483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176798.796489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176798.796499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176798.796505:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176798.796510:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176798.796517:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176798.796523:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176798.796527:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176798.796531:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176798.796534:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176798.796538:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176798.796540:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176798.796602:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176798.796607:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176798.796611:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176799.436463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176799.436472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176799.436475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176799.436481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176799.436486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176799.436490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176799.436493:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176799.436496:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176799.436499:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176799.820456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176799.820461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176799.820466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176799.820470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176799.820474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176799.820476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176799.820537:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176799.820542:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176799.820545:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176800.460528:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176800.460540:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176800.460542:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176800.460550:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176800.460554:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176800.460557:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176800.460560:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176800.460562:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176800.460564:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176800.780464:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306147584, at 4306147648 00000100:00000010:0.0:1786176800.780492:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cf60. 00010000:00000001:0.0:1786176800.844511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176800.844522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176800.844536:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176800.844540:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176800.844546:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176800.844548:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176800.844629:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176800.844633:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176800.844636:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176801.484430:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176801.484439:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176801.484442:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176801.484448:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176801.484455:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176801.484459:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.484462:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176801.484465:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.484470:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176801.868515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176801.868525:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176801.868531:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176801.868547:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176801.868557:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176801.868568:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176801.868576:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176801.868578:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176801.868581:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176801.868584:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176801.868590:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176801.868593:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176801.868595:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176801.868597:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176801.868598:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176801.868600:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176801.868602:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176801.868604:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176801.868668:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176801.868671:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176801.868674:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176801.996479:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176801.996488:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176801.996490:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176801.996493:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176801.996497:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176801.996503:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11476 00000400:00000200:0.0:1786176801.996510:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176801.996516:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176801.996518:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176801.996521:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996523:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176801.996525:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996527:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176801.996530:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176801.996532:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176801.996534:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176801.996536:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176801.996538:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176801.996540:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176801.996542:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996543:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176801.996544:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996546:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176801.996548:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176801.996550:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996552:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176801.996554:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176801.996557:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176801.996559:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.996561:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176801.996563:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176801.996571:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176801.996574:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176801.996577:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176801.996578:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.996580:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176801.996582:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176801.996586:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176801.996592:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176801.996597:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176801.996604:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176801.996624:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176801.996628:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176801.996632:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176801.996649:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176801.996652:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176801.996654:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176801.996655:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176801.996656:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176801.996659:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176801.996661:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11476 00000400:00000200:0.0:1786176801.996663:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176801.996665:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176801.996667:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176801.996669:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996670:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176801.996672:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996674:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176801.996675:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176801.996676:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176801.996678:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176801.996679:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176801.996680:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176801.996682:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176801.996683:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996684:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176801.996686:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996687:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176801.996688:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176801.996690:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996691:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176801.996692:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176801.996696:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176801.996697:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.996698:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176801.996701:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176801.996704:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.996705:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176801.996707:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176801.996709:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176801.996710:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176801.996711:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.996712:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176801.996713:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176801.996715:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176801.996718:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176801.996721:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176801.996727:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176801.996737:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176801.996738:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176801.996741:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.996745:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786176801.996747:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176801.996756:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786176801.996757:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1786176801.996759:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176801.996762:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1786176801.996763:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1786176801.996774:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176801.996778:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176801.996785:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11476 00000100:00000040:2.0:1786176801.996786:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1786176801.996791:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176801.996793:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176801.996795:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176801.996797:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996803:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176801.996811:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996820:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176801.996832:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786176801.996833:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1786176801.996834:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000100:00000001:2.0:1786176801.996839:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176801.996840:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176801.996846:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 02000000:00000001:2.0:1786176801.996849:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1786176801.996850:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176801.996853:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 02000000:00000001:2.0:1786176801.996854:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176801.996863:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176801.996868:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.996874:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1786176801.996874:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235537920:192.168.202.132@tcp:250:kworker.0 00000100:00000040:0.0:1786176801.996886:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176801.996886:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1786176801.996893:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:2.0:1786176801.996896:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:0.0:1786176801.996898:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 02000000:00000001:2.0:1786176801.996899:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176801.996907:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:2.0:1786176801.996907:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1786176801.996915:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176801.996916:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176801.996918:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176801.996920:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176801.996921:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 00000100:00000001:0.0:1786176801.996924:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176801.996926:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176801.996928:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.996929:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176801.996930:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176801.996932:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176801.996935:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176801.996938:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176801.996939:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176801.996943:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786176801.996945:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176801.996948:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1786176801.996951:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 02000000:00000001:0.0:1786176801.996952:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176801.996966:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000400:00000010:2.0:1786176801.996969:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000001:0.0:1786176801.996971:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176801.996977:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1786176801.996979:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235537920, portal 25 00000100:00000001:2.0:1786176801.996990:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176801.996994:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794213248 : -110371915338368 : ffff9b9e09316d80) 00000100:00000001:0.0:1786176801.996997:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176801.997002:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176801.997007:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786176801.997009:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.997065:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1786176801.997068:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176801.997073:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1786176801.997081:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00080000:0.0:1786176801.997082:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176801.997086:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:2.0:1786176801.997087:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00080000:0.0:1786176801.997095:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1786176801.997097:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235537920, offset 0 00000100:00000001:0.0:1786176801.997099:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176801.997106:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11476 00000400:00000200:2.0:1786176801.997113:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786176801.997118:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176801.997121:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176801.997123:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176801.997125:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:2.0:1786176801.997132:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786176801.997136:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176801.997139:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.997141:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176801.997142:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176801.997146:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000400:00000200:2.0:1786176801.997146:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000001:0.0:1786176801.997165:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176801.997168:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176801.997170:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000200:2.0:1786176801.997172:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1786176801.997174:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000200:2.0:1786176801.997184:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:0.0:1786176801.997185:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.997193:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786176801.997196:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1786176801.997197:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.997205:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786176801.997207:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000040:0.0:1786176801.997213:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000400:00000200:2.0:1786176801.997221:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426269:426269:256:4294967295] 192.168.202.132@tcp LPNI seq info [426231:426231:8:4294967295] 00000100:00080000:0.0:1786176801.997226:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176801.997232:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176801.997234:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:0.0:1786176801.997242:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176801.997247:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176801.997250:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.997255:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176801.997260:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000200:2.0:1786176801.997266:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000008:00000001:0.0:1786176801.997268:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.997277:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176801.997280:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176801.997285:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000800:00000200:2.0:1786176801.997285:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 02000000:00000001:0.0:1786176801.997299:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176801.997300:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.997301:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176801.997305:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000010:2.0:1786176801.997305:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 02000000:00000001:0.0:1786176801.997310:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176801.997315:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000800:00000200:2.0:1786176801.997322:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786176801.997324:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176801.997332:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.0:1786176801.997333:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000001:0.0:1786176801.997346:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176801.997348:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176801.997354:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:0.0:1786176801.997390:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1:1786176801.997408:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000001:0.0:1786176801.997410:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176801.997416:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176801.997421:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176801.997423:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176801.997425:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176801.997429:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:2.0:1786176801.997435:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00080000:0.0:1786176801.997436:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11476 00000800:00000010:2.0:1786176801.997444:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:0.0:1786176801.997455:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176801.997462:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:2.0:1786176801.997462:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:0.0:1786176801.997472:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786176801.997479:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1786176801.997480:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176801.997486:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176801.997489:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:2.0:1786176801.997490:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000100:00000001:0.0:1786176801.997492:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:2.0:1786176801.997496:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000020:00000001:0.0:1786176801.997498:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176801.997501:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176801.997505:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1786176801.997506:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:0.0:1786176801.997509:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176801.997513:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000200:2.0:1786176801.997518:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1786176801.997520:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176801.997537:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176801.997539:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1786176801.997545:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176801.997546:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786176801.997556:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176801.997559:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.997562:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786176801.997566:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786176801.997571:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1786176801.997573:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000400:00000200:2.0:1786176801.997580:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:0.0:1786176801.997585:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176801.997587:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306152705 00000100:00000001:0.0:1786176801.997595:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176801.997596:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cc00. 02000000:00000001:0.0:1786176801.997601:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786176801.997607:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1786176801.997608:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176801.997614:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.997616:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176801.997619:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306152705 00000008:00100000:0.0:1786176801.997620:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000010:2.0:1786176801.997625:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c2a0. 00000008:00000001:0.0:1786176801.997627:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.997630:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.997630:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786176801.997640:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1786176801.997641:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176801.997652:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000001:2.0:1786176801.997652:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.997658:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1786176801.997659:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176801.997664:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176801.997672:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1786176801.997673:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176801/real 1786176801] req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786176801.997681:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176801.997686:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176801.997695:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176801.997702:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1786176801.997702:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176801.997705:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176801.997708:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176801.997710:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000100:00100000:0.0:1786176801.997712:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1786176801.997714:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176801.997723:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1786176801.997729:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1786176801.997731:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176801.997734:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176801.997746:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176801.997761:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176801.997766:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176801.997776:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11486) 00000100:00000001:2.0:1786176801.997778:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176801.997782:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.997785:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176801.997787:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176801.997790:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.997796:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176801.997800:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.997805:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176801.997808:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.997809:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176801.997810:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.997812:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176801.997813:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176801.997816:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176801.997819:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176801.997820:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176801.997823:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176801.997830:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235537920:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176801.997833:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176801.997836:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.997840:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176801.997841:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176801.997843:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176801.997845:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176801.997846:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176801.997848:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235538048:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176801.997850:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176801.997851:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176801.997853:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176801.997854:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176801.997857:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176801.997858:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176801.997861:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176801.997864:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176801.997866:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176801.997869:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235538048, portal 10 00000100:00000001:2.0:1786176801.997870:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176801.997871:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215040 : -110371915336576 : ffff9b9e09317480) 00000100:00000040:2.0:1786176801.997877:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.997881:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176801.997883:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176801.997885:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176801.997886:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235538048, offset 0 00000400:00000200:2.0:1786176801.997889:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176801.997892:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176801.997895:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176801.997898:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176801.997900:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176801.997902:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176801.997903:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176801.997906:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426270:426270:256:4294967295] 192.168.202.132@tcp LPNI seq info [426232:426232:8:4294967295] 00000400:00000200:2.0:1786176801.997909:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176801.997914:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176801.997917:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176801.997919:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176801.997922:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176801.997924:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176801.997927:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176801.997929:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786176801.997932:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176801.997934:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176801.997936:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176801.997938:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176801.997941:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176801.997942:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176801.997943:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176801.997946:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.997951:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176801.997953:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.997957:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.997958:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176801.997960:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176801.997964:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786176801.997966:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176801.997968:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306148868 00000100:00000001:2.0:1786176801.997970:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306152705 00000100:00000001:2.0:1786176801.997972:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306148868 00000100:00000001:2.0:1786176801.997974:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306152705 00000100:00000001:2.0:1786176801.997975:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176801.997976:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.997977:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176801.997979:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176801/real 1786176801] req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176801.997984:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176801.997985:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176801.997986:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176801.997987:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176801.997989:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176801.997990:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176801.997992:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176801.997997:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998001:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176801.998002:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998003:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176801.998004:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176801.998006:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998010:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176801.998012:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998016:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176801.998019:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998020:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176801.998021:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998022:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176801.998024:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176801.998026:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176801.998028:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176801.998029:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176801.998031:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176801.998036:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235538048:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176801.998038:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998040:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176801.998042:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316d80 x1872930235537920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998046:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176801.998049:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176801.998051:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176801.998053:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176801.998055:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176801.998057:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786176801.998059:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176801.998061:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:2.0:1786176801.998065:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176801.998067:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176801.998068:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316d80. 00000100:00000001:2.0:1786176801.998071:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176801.998072:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998073:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176801.998075:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317480 x1872930235538048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998084:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176801.998085:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176801.998086:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176801.998088:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176801.998088:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176801.998089:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176801.998090:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176801.998091:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:2.0:1786176801.998092:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176801.998093:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176801.998094:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317480. 00000100:00000001:2.0:1786176801.998095:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176801.998096:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998097:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998104:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176801.998106:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.998108:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176801.998109:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176801.998110:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176801.998112:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998116:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176801.998117:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176801.998119:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176801.998120:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176801.998121:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176801.998123:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235538176:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176801.998125:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176801.998127:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176801.998128:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176801.998129:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176801.998131:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176801.998132:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176801.998135:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176801.998137:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176801.998139:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176801.998141:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235538176, portal 10 00000100:00000001:2.0:1786176801.998142:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176801.998143:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794203392 : -110371915348224 : ffff9b9e09314700) 00000100:00000040:2.0:1786176801.998147:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998172:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176801.998174:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176801.998176:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176801.998178:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235538176, offset 0 00000400:00000200:2.0:1786176801.998181:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176801.998183:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176801.998186:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176801.998188:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176801.998190:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176801.998192:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176801.998194:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176801.998196:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426271:426271:256:4294967295] 192.168.202.132@tcp LPNI seq info [426233:426233:8:4294967295] 00000400:00000200:2.0:1786176801.998199:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176801.998203:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176801.998205:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176801.998207:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176801.998210:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176801.998212:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176801.998215:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176801.998217:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786176801.998220:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176801.998221:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176801.998223:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176801.998226:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176801.998228:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176801.998230:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176801.998231:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176801.998234:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998240:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176801.998241:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998246:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.998247:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176801.998248:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176801.998251:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786176801.998253:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176801.998254:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306148868 00000100:00000001:2.0:1786176801.998255:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306152705 00000100:00000001:2.0:1786176801.998257:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306148868 00000100:00000001:2.0:1786176801.998259:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306152705 00000100:00000001:2.0:1786176801.998260:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176801.998261:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.998263:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176801.998265:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176801/real 1786176801] req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176801.998269:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176801.998270:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176801.998271:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176801.998272:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176801.998273:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176801.998275:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176801.998277:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176801.998282:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998287:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176801.998288:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998290:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176801.998291:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176801.998293:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998297:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176801.998299:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998303:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176801.998304:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998305:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176801.998306:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998307:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176801.998309:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176801.998310:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176801.998312:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176801.998313:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176801.998316:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176801.998320:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235538176:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176801.998322:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176801.998325:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998330:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176801.998331:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176801.998333:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176801.998335:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176801.998336:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176801.998338:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235538304:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176801.998341:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176801.998342:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176801.998343:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176801.998344:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176801.998345:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176801.998347:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176801.998349:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176801.998351:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176801.998353:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176801.998355:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235538304, portal 4 00000100:00000001:2.0:1786176801.998376:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176801.998377:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794204288 : -110371915347328 : ffff9b9e09314a80) 00000100:00000040:2.0:1786176801.998382:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998386:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176801.998388:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176801.998390:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176801.998392:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235538304, offset 0 00000400:00000200:2.0:1786176801.998394:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176801.998397:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176801.998399:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176801.998401:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176801.998403:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176801.998405:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176801.998406:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176801.998408:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426272:426272:256:4294967295] 192.168.202.132@tcp LPNI seq info [426234:426234:8:4294967295] 00000400:00000200:2.0:1786176801.998411:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176801.998415:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176801.998419:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176801.998421:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176801.998424:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176801.998426:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176801.998429:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176801.998431:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786176801.998434:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176801.998435:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176801.998437:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176801.998439:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176801.998442:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176801.998443:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176801.998444:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176801.998447:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998452:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176801.998454:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998459:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.998460:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176801.998462:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176801.998464:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786176801.998466:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176801.998467:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306148869 00000100:00000001:2.0:1786176801.998469:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306152705 00000100:00000001:2.0:1786176801.998471:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306148869 00000100:00000001:2.0:1786176801.998472:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306152705 00000100:00000001:2.0:1786176801.998473:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176801.998474:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.998476:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176801.998478:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176801/real 1786176801] req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176801.998484:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176801.998485:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176801.998486:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176801.998488:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176801.998489:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176801.998490:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176801.998492:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176801.998497:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998501:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176801.998502:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998504:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176801.998505:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176801.998507:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998511:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176801.998513:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998517:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176801.998518:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998519:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176801.998520:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998521:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176801.998522:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176801.998524:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176801.998526:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176801.998527:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176801.998530:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176801.998536:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235538304:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176801.998538:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176801.998541:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998545:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176801.998546:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176801.998547:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176801.998548:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176801.998549:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176801.998551:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09315500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235538432:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176801.998553:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176801.998554:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176801.998555:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176801.998556:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176801.998558:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786176801.998559:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176801.998561:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176801.998563:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176801.998565:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176801.998566:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235538432, portal 4 00000100:00000001:2.0:1786176801.998568:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176801.998569:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794206976 : -110371915344640 : ffff9b9e09315500) 00000100:00000040:2.0:1786176801.998573:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998577:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176801.998579:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176801.998580:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176801.998582:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235538432, offset 0 00000400:00000200:2.0:1786176801.998584:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176801.998586:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176801.998589:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176801.998591:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176801.998592:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176801.998594:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176801.998596:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176801.998598:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426273:426273:256:4294967295] 192.168.202.132@tcp LPNI seq info [426235:426235:8:4294967295] 00000400:00000200:2.0:1786176801.998600:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176801.998604:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176801.998607:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176801.998608:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176801.998611:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176801.998612:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786176801.998615:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176801.998617:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786176801.998619:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176801.998621:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176801.998623:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176801.998625:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176801.998627:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176801.998628:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176801.998629:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176801.998631:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998637:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176801.998638:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998643:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.998644:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176801.998645:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176801.998647:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786176801.998649:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176801.998651:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306148871 00000100:00000001:2.0:1786176801.998653:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306152705 00000100:00000001:2.0:1786176801.998655:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306148871 00000100:00000001:2.0:1786176801.998656:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306152705 00000100:00000001:2.0:1786176801.998657:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176801.998658:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.998659:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176801.998661:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176801/real 1786176801] req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176801.998666:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176801.998667:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176801.998667:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176801.998669:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176801.998670:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176801.998671:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176801.998673:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176801.998677:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998682:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176801.998683:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998684:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176801.998685:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176801.998687:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998691:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176801.998693:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998697:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176801.998699:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998700:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176801.998701:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998702:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176801.998704:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176801.998705:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176801.998707:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176801.998708:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176801.998711:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176801.998716:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09315500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235538432:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176801.998718:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998719:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176801.998721:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314700 x1872930235538176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998724:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176801.998725:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176801.998726:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176801.998728:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176801.998728:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176801.998729:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176801.998730:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176801.998731:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:2.0:1786176801.998733:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176801.998734:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176801.998735:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314700. 00000100:00000001:2.0:1786176801.998738:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176801.998739:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998740:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176801.998742:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314a80 x1872930235538304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998751:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176801.998752:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176801.998753:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786176801.998755:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176801.998755:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176801.998756:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176801.998757:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176801.998758:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:2.0:1786176801.998759:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176801.998760:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176801.998761:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314a80. 00000100:00000001:2.0:1786176801.998762:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176801.998763:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998763:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176801.998765:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09315500 x1872930235538432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176801.998769:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176801.998770:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176801.998770:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786176801.998771:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176801.998772:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176801.998773:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176801.998774:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176801.998775:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:2.0:1786176801.998776:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176801.998776:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176801.998777:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09315500. 00000100:00000001:2.0:1786176801.998778:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176801.998779:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998780:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176801.998782:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176801.998783:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176801.998784:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176801.998786:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176802.508500:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176802.508512:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176802.508514:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176802.508522:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176802.508526:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176802.508530:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176802.508533:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176802.508535:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176802.508538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176802.892494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176802.892503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176802.892516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176802.892521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176802.892528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176802.892533:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176802.892606:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176802.892610:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176802.892614:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176803.532457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176803.532468:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176803.532471:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176803.532485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176803.532490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176803.532494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176803.532496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176803.532499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176803.532502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176803.916471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176803.916476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176803.916481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176803.916484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176803.916489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176803.916491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176803.916566:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176803.916569:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176803.916572:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176804.556536:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176804.556545:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176804.556547:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176804.556553:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176804.556558:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176804.556563:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176804.556566:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176804.556570:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176804.556573:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176804.940472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176804.940479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176804.940486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176804.940490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176804.940497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176804.940501:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176804.940563:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176804.940566:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176804.940568:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176805.580466:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176805.580474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176805.580477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176805.580484:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176805.580489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176805.580492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176805.580496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176805.580499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176805.580502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176805.900521:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306152705, at 4306152768 00000100:00000010:0.0:1786176805.900539:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cc00. 00010000:00000001:0.0:1786176805.964503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176805.964509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176805.964520:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176805.964525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176805.964529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176805.964530:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176805.964614:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176805.964618:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176805.964621:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176806.604502:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176806.604514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176806.604518:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176806.604525:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176806.604531:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176806.604536:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176806.604539:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176806.604543:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176806.604547:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176806.988511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176806.988520:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176806.988527:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176806.988529:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176806.988533:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176806.988534:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176806.988615:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176806.988622:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176806.988626:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786176807.116532:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176807.116546:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786176807.116550:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176807.116555:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176807.116561:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176807.116571:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11481 00000400:00000200:2.0:1786176807.116582:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176807.116599:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176807.116603:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176807.116608:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.116613:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176807.116616:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.116618:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176807.116624:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176807.116626:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786176807.116630:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176807.116634:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786176807.116639:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786176807.116642:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786176807.116645:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.116648:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176807.116650:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.116653:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176807.116663:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786176807.116667:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176807.116670:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176807.116672:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176807.116676:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176807.116679:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.116681:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176807.116684:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176807.116696:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735180. 00000020:00000040:2.0:1786176807.116699:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786176807.116703:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176807.116705:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.116708:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176807.116711:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176807.116715:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176807.116723:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:2.0:1786176807.116730:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176807.116740:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735180 x1872930235538560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176807.116750:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176807.116754:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176807.116759:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ba0f078] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735180 x1872930235538560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176807.116785:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176807.116788:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176807.116792:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786176807.116794:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176807.116796:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176807.116800:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176807.116803:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11481 00000400:00000200:2.0:1786176807.116808:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176807.116815:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176807.116818:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176807.116822:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.116825:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176807.116826:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.116830:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176807.116834:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176807.116842:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786176807.116850:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176807.116857:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786176807.116864:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786176807.116872:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786176807.116891:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.116900:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176807.116915:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.116920:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176807.116926:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786176807.116939:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176807.116944:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176807.116953:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176807.116963:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176807.116971:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.116975:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176807.116983:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176807.116988:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.116992:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176807.116998:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734e00. 00000020:00000040:2.0:1786176807.117004:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176807.117009:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176807.117010:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.117012:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176807.117015:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176807.117017:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176807.117022:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aabc00. 00000100:00000001:2.0:1786176807.117027:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176807.117040:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734e00 x1872930235538688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176807.117066:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176807.117078:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176807.117099:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001f62212] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734e00 x1872930235538688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176807.117127:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176807.117132:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176807.117136:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786176807.117137:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176807.117143:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176807.117157:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176807.117162:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11481 00000400:00000200:2.0:1786176807.117173:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176807.117183:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176807.117192:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176807.117197:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117201:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176807.117207:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117215:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176807.117220:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176807.117224:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786176807.117229:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176807.117232:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786176807.117234:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786176807.117237:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786176807.117249:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117251:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176807.117255:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117259:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176807.117263:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786176807.117266:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117268:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176807.117269:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176807.117273:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176807.117276:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.117280:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176807.117282:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176807.117287:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.117290:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176807.117301:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000020:00000040:2.0:1786176807.117305:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176807.117307:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176807.117308:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.117314:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176807.117317:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176807.117322:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176807.117328:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786176807.117333:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176807.117343:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235538816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1786176807.117387:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176807.117391:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1786176807.117430:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176807.117437:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176807.117452:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235538816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176807.117476:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176807.117484:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176807.117491:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786176807.117495:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176807.117499:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176807.117513:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176807.117518:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11481 00000400:00000200:2.0:1786176807.117524:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176807.117530:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176807.117532:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176807.117542:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117554:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176807.117561:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117564:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176807.117569:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176807.117575:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786176807.117580:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176807.117587:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786176807.117594:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786176807.117604:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786176807.117617:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117624:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176807.117632:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117641:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176807.117652:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786176807.117661:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117672:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176807.117680:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176807.117684:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176807.117688:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.117690:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176807.117694:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176807.117701:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.117707:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176807.117718:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000020:00000040:2.0:1786176807.117723:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176807.117727:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176807.117730:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.117735:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176807.117738:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176807.117742:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176807.117747:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000001:2.0:1786176807.117757:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176807.117773:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235538944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176807.117826:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176807.117837:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176807.117853:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235538944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176807.117864:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176807.117869:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176807.117879:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786176807.117882:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176807.117884:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176807.117893:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176807.117898:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11481 00000400:00000200:2.0:1786176807.117909:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176807.117915:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176807.117921:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176807.117925:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117929:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176807.117933:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117939:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176807.117945:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176807.117948:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786176807.117955:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176807.117960:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786176807.117966:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786176807.117973:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786176807.117984:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.117990:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176807.117994:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176807.118000:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176807.118004:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786176807.118014:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176807.118017:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176807.118018:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176807.118024:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176807.118031:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.118035:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176807.118038:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176807.118046:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.118053:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176807.118067:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000020:00000040:2.0:1786176807.118076:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176807.118085:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176807.118094:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176807.118118:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176807.118122:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176807.118131:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176807.118138:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000001:2.0:1786176807.118144:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176807.118157:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235539072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176807.118182:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176807.118190:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176807.118201:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235539072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176807.118215:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176807.118222:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11491) 00000800:00000200:1.1:1786176807.118412:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176807.118415:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176807.118689:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176807.118691:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176807.118885:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176807.118887:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176807.119088:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176807.119090:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1786176807.628463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176807.628475:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176807.628478:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176807.628483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176807.628487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176807.628492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176807.628495:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176807.628499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176807.628502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176808.012461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176808.012466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176808.012474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176808.012477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176808.012482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176808.012485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176808.012558:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176808.012563:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176808.012566:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176808.652546:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176808.652553:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176808.652554:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176808.652559:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176808.652562:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176808.652566:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176808.652568:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176808.652570:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176808.652572:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176809.036518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176809.036525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176809.036535:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176809.036541:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176809.036545:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176809.036551:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176809.036558:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176809.036562:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176809.036567:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176809.036572:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176809.036577:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176809.036578:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176809.036649:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176809.036654:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176809.036658:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176809.676475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176809.676489:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176809.676491:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176809.676499:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176809.676505:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176809.676511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176809.676514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176809.676518:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176809.676522:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176810.060477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176810.060482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176810.060492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176810.060496:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176810.060504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176810.060508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176810.060604:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176810.060613:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176810.060618:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176810.700412:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176810.700420:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176810.700423:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176810.700429:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176810.700434:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176810.700438:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176810.700441:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176810.700445:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176810.700448:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176811.020474:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306157825, at 4306157888 00000100:00000010:0.0:1786176811.020494:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ba0. 00010000:00000001:0.0:1786176811.084428:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176811.084433:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176811.084438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176811.084442:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176811.084448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176811.084450:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176811.084504:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176811.084510:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176811.084513:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176811.724409:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176811.724420:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176811.724422:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176811.724428:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176811.724432:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176811.724436:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176811.724440:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176811.724444:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176811.724447:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176812.108449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176812.108460:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176812.108466:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176812.108474:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176812.108483:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176812.108487:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176812.108490:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176812.108493:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176812.108500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176812.108503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176812.108507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176812.108509:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176812.108511:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176812.108512:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176812.108515:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176812.108516:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176812.108518:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176812.108520:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176812.108578:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176812.108583:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176812.108588:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176812.236464:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176812.236473:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176812.236475:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176812.236479:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176812.236484:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176812.236492:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11486 00000400:00000200:0.0:1786176812.236501:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176812.236508:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176812.236512:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176812.236515:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.236518:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176812.236521:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.236524:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176812.236528:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176812.236531:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176812.236533:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176812.236537:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176812.236540:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176812.236542:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176812.236545:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.236547:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176812.236549:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.236551:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176812.236555:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176812.236558:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176812.236561:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176812.236563:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176812.236567:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176812.236570:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.236572:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176812.236575:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176812.236584:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176812.236588:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176812.236592:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176812.236593:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.236596:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176812.236599:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176812.236603:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176812.236612:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176812.236619:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176812.236628:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235539200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176812.236638:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176812.236642:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176812.236647:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235539200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176812.236672:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176812.236675:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176812.236679:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176812.236681:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176812.236683:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176812.236686:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176812.236689:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11486 00000400:00000200:0.0:1786176812.236694:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176812.236699:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176812.236702:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176812.236706:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.236709:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176812.236712:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.236714:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176812.236717:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176812.236719:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176812.236721:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176812.236723:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176812.236726:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176812.236728:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176812.236731:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.236734:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176812.236736:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.236738:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176812.236740:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176812.236743:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176812.236747:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176812.236752:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176812.236759:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176812.236763:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.236766:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176812.236772:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176812.236784:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.236794:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176812.236805:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176812.236815:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176812.236826:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176812.236833:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.236839:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176812.236843:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176812.236846:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176812.236854:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176812.236861:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176812.236871:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235539328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176812.236892:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176812.236898:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176812.236908:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235539328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176812.236925:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176812.236930:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176812.236940:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176812.236945:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176812.236949:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176812.236954:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176812.236962:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11486 00000400:00000200:0.0:1786176812.236977:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176812.236989:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176812.236997:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176812.237008:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.237013:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176812.237026:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.237038:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176812.237043:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176812.237049:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176812.237089:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176812.237097:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176812.237106:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176812.237113:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176812.237129:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.237141:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176812.237150:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.237158:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176812.237166:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000200:1.1:1786176812.237172:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:0.0:1786176812.237177:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:1.1:1786176812.237180:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1786176812.237186:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176812.237194:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176812.237204:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176812.237217:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.237231:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176812.237238:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176812.237255:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.237260:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176812.237271:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176812.237283:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176812.237290:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176812.237294:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.237301:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176812.237305:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176812.237313:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176812.237334:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176812.237343:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176812.237405:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235539456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176812.237448:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176812.237459:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176812.237475:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235539456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176812.237539:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176812.237549:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176812.237563:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176812.237568:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176812.237572:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176812.237587:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176812.237597:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11486 00000400:00000200:0.0:1786176812.237618:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176812.237633:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176812.237642:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176812.237662:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.237674:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176812.237686:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.237699:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176812.237707:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176812.237715:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176812.237723:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176812.237730:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176812.237741:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176812.237749:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176812.237764:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.237768:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176812.237771:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.237775:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176812.237789:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176812.237802:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176812.237813:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176812.237825:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176812.237835:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176812.237846:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.237858:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176812.237866:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176812.237874:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.237880:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176812.237892:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176812.237903:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176812.237910:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176812.237914:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.237920:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176812.237925:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176812.237932:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176812.237943:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176812.237956:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176812.237970:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235539584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176812.237998:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176812.238009:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176812.238026:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235539584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176812.238087:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176812.238098:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1786176812.238098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176812.238102:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00080000:0.0:1786176812.238111:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176812.238119:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176812.238125:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176812.238133:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176812.238148:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11486 00000400:00000200:0.0:1786176812.238160:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176812.238168:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176812.238175:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176812.238189:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.238200:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176812.238205:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.238211:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176812.238217:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176812.238221:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176812.238227:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176812.238231:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176812.238240:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176812.238251:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176812.238265:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.238271:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176812.238276:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176812.238285:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176812.238293:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176812.238303:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176812.238310:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176812.238315:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176812.238322:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176812.238329:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.238334:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176812.238340:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176812.238348:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.238355:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176812.238382:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176812.238385:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176812.238393:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176812.238396:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176812.238405:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176812.238408:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176812.238411:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176812.238419:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176812.238431:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176812.238444:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235539712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176812.238513:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176812.238520:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176812.238534:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235539712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176812.238562:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176812.238572:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11496) 00000800:00000200:1.1:1786176812.238769:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176812.238771:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176812.239180:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176812.239183:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176812.239466:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176812.239467:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.0:1786176812.684510:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786176812.684518:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786176812.684524:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1786176812.684710:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176812.684728:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786176812.684734:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176812.684736:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176812.748420:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176812.748430:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176812.748433:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176812.748440:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176812.748444:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176812.748449:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176812.748452:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176812.748455:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176812.748458:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176813.132428:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176813.132440:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176813.132445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176813.132446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176813.132449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176813.132450:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176813.132501:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176813.132507:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176813.132513:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176813.772415:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176813.772427:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176813.772430:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176813.772438:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176813.772443:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176813.772448:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176813.772453:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176813.772456:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176813.772459:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176814.156477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176814.156485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176814.156491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176814.156496:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176814.156501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176814.156504:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176814.156594:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176814.156597:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176814.156600:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176814.796448:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176814.796459:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176814.796462:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176814.796471:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176814.796478:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176814.796483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176814.796487:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176814.796492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176814.796496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176815.180469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176815.180477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176815.180490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176815.180497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176815.180501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176815.180503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176815.180597:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176815.180602:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176815.180605:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176815.820424:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176815.820433:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176815.820435:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176815.820442:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176815.820448:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176815.820453:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176815.820456:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176815.820460:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176815.820464:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176816.140491:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306162944, at 4306163008 00000100:00000010:0.0:1786176816.140510:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62900. 00010000:00000001:0.0:1786176816.204445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176816.204452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176816.204456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176816.204458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176816.204463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176816.204465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176816.204539:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176816.204545:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176816.204549:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176816.844437:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176816.844447:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176816.844450:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176816.844456:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176816.844461:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176816.844465:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176816.844468:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176816.844472:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176816.844475:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176817.228496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176817.228502:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176817.228513:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176817.228517:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176817.228528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176817.228537:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176817.228612:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176817.228617:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176817.228620:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786176817.356516:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176817.356525:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786176817.356528:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176817.356530:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176817.356534:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176817.356541:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11491 00000400:00000200:2.0:1786176817.356548:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176817.356553:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176817.356555:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176817.356557:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.356560:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176817.356561:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.356563:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176817.356568:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176817.356570:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786176817.356573:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176817.356577:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786176817.356579:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786176817.356583:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786176817.356586:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.356588:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176817.356590:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.356593:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176817.356596:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786176817.356599:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176817.356602:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176817.356605:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176817.356609:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176817.356612:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.356614:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176817.356617:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176817.356624:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000020:00000040:2.0:1786176817.356628:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786176817.356632:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176817.356634:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.356637:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176817.356639:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176817.356643:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176817.356648:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786176817.356653:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176817.356662:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235539840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176817.356671:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176817.356675:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176817.356678:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235539840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176817.356698:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176817.356700:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176817.356703:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786176817.356705:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176817.356706:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176817.356709:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176817.356712:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11491 00000400:00000200:2.0:1786176817.356716:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176817.356720:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176817.356723:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176817.356725:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.356727:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176817.356729:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.356730:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176817.356732:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176817.356733:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786176817.356737:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176817.356742:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786176817.356749:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786176817.356756:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786176817.356770:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.356779:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176817.356786:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.356794:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176817.356802:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786176817.356812:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176817.356819:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176817.356822:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176817.356829:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176817.356841:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.356847:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176817.356851:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176817.356858:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.356864:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176817.356872:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 00000020:00000040:2.0:1786176817.356875:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176817.356881:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176817.356883:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.356889:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176817.356891:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176817.356901:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176817.356908:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:2.0:1786176817.356917:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176817.356929:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235539968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176817.356947:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176817.356955:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176817.356961:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235539968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176817.356992:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176817.357013:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176817.357037:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786176817.357043:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176817.357047:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176817.357053:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176817.357063:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11491 00000400:00000200:2.0:1786176817.357076:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176817.357084:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176817.357087:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176817.357090:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357092:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176817.357093:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357094:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176817.357096:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176817.357098:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786176817.357099:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176817.357104:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786176817.357117:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786176817.357120:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786176817.357128:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357137:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176817.357143:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357148:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176817.357153:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786176817.357160:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357166:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176817.357181:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176817.357190:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176817.357200:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.357211:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176817.357217:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176817.357225:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.357229:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176817.357233:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 00000020:00000040:2.0:1786176817.357238:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176817.357248:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176817.357251:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.357257:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176817.357259:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176817.357268:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176817.357291:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:2.0:1786176817.357302:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176817.357312:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235540096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176817.357334:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1786176817.357335:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786176817.357341:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176817.357342:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000040:2.0:1786176817.357432:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235540096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176817.357468:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176817.357473:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176817.357479:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786176817.357485:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176817.357486:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176817.357490:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176817.357495:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11491 00000400:00000200:2.0:1786176817.357506:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176817.357511:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176817.357519:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176817.357525:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357529:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176817.357532:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357535:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176817.357540:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176817.357541:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786176817.357543:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176817.357544:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786176817.357547:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786176817.357557:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786176817.357561:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357563:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176817.357566:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357579:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176817.357582:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786176817.357588:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357591:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176817.357601:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176817.357605:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176817.357617:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.357621:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176817.357625:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176817.357629:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.357633:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176817.357636:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000020:00000040:2.0:1786176817.357641:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176817.357651:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176817.357654:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.357659:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176817.357663:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176817.357668:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176817.357677:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:2.0:1786176817.357680:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176817.357689:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235540224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176817.357706:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176817.357712:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176817.357719:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235540224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176817.357731:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176817.357741:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176817.357755:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786176817.357758:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176817.357763:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176817.357770:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176817.357775:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11491 00000400:00000200:2.0:1786176817.357780:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176817.357786:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176817.357795:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176817.357801:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357808:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176817.357812:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357819:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176817.357826:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176817.357829:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786176817.357835:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176817.357837:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786176817.357847:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786176817.357855:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786176817.357864:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357873:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176817.357876:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357884:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176817.357887:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786176817.357894:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176817.357900:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176817.357903:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176817.357906:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176817.357911:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.357918:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176817.357923:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176817.357934:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.357940:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176817.357946:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000020:00000040:2.0:1786176817.357956:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176817.357963:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176817.357966:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176817.357970:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176817.357972:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176817.357975:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176817.357985:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:2.0:1786176817.357992:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176817.358017:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235540352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176817.358025:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176817.358029:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176817.358036:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235540352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176817.358057:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176817.358071:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11501) 00000800:00000200:1.1:1786176817.358327:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176817.358330:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176817.358694:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176817.358697:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176817.359038:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176817.359040:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176817.359348:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176817.359351:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:3.0:1786176817.868506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176817.868513:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176817.868515:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176817.868521:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176817.868526:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176817.868530:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176817.868534:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176817.868536:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176817.868541:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176818.252508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176818.252514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176818.252523:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176818.252528:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176818.252536:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176818.252538:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176818.252606:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176818.252609:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176818.252613:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176818.892458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176818.892467:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176818.892470:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176818.892476:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176818.892481:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176818.892486:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176818.892489:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176818.892492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176818.892498:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176819.276483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176819.276489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176819.276494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176819.276497:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176819.276499:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176819.276501:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176819.276504:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176819.276506:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176819.276507:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176819.276509:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176819.276511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176819.276512:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176819.276571:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176819.276574:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176819.276577:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176819.916681:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176819.916692:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176819.916695:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176819.916702:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176819.916707:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176819.916711:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176819.916714:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176819.916718:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176819.916722:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176820.300446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176820.300451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176820.300459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176820.300463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176820.300469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176820.300472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176820.300534:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176820.300538:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176820.300542:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176820.940500:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176820.940509:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176820.940512:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176820.940519:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176820.940523:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176820.940528:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176820.940531:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176820.940534:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176820.940538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176821.260461:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306168065, at 4306168128 00000100:00000010:0.0:1786176821.260477:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62060. 00010000:00000001:0.0:1786176821.324457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176821.324461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176821.324467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176821.324470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176821.324473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176821.324476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176821.324528:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176821.324531:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176821.324534:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176821.964828:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176821.964837:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176821.964840:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176821.964846:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176821.964851:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176821.964855:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176821.964858:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176821.964861:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176821.964865:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176822.348449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176822.348457:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176822.348461:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176822.348465:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176822.348468:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176822.348470:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176822.348471:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176822.348473:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176822.348477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176822.348478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176822.348481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176822.348482:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176822.348483:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176822.348484:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176822.348486:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176822.348487:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176822.348488:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176822.348489:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176822.348540:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176822.348544:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176822.348547:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176822.476494:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176822.476503:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176822.476505:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176822.476508:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176822.476512:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176822.476518:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11496 00000400:00000200:0.0:1786176822.476524:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176822.476529:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176822.476532:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176822.476534:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476536:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176822.476539:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476541:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176822.476544:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176822.476545:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176822.476548:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176822.476550:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176822.476552:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176822.476553:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176822.476555:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476556:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176822.476558:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476559:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176822.476561:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176822.476563:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476565:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176822.476567:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176822.476570:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176822.476572:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.476574:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176822.476576:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176822.476584:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786176822.476586:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176822.476589:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176822.476591:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.476593:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176822.476596:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176822.476599:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176822.476606:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786176822.476611:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176822.476618:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235540480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176822.476626:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176822.476630:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176822.476633:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235540480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176822.476661:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176822.476664:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176822.476666:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176822.476667:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176822.476668:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176822.476670:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176822.476672:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11496 00000400:00000200:0.0:1786176822.476674:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176822.476677:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176822.476678:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176822.476680:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476682:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176822.476683:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476685:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176822.476687:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176822.476688:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176822.476689:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176822.476691:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176822.476692:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176822.476694:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176822.476696:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476697:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176822.476699:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476700:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176822.476701:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176822.476703:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476707:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176822.476709:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176822.476714:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176822.476717:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.476720:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176822.476723:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176822.476727:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.476729:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176822.476733:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786176822.476736:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176822.476741:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176822.476743:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.476746:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176822.476749:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176822.476750:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176822.476755:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176822.476759:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176822.476767:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235540608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176822.476779:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176822.476785:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176822.476794:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235540608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176822.476810:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176822.476817:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176822.476826:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176822.476830:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176822.476834:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176822.476838:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176822.476841:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11496 00000400:00000200:0.0:1786176822.476847:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176822.476853:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176822.476856:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176822.476861:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476864:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176822.476868:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476882:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176822.476888:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176822.476890:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176822.476893:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176822.476896:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176822.476898:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176822.476900:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176822.476904:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476907:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176822.476909:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476912:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176822.476917:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176822.476922:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176822.476924:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176822.476927:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176822.476931:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176822.476935:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.476938:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176822.476942:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176822.476948:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.476972:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176822.476978:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786176822.476982:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176822.476986:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176822.476990:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.476991:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176822.476994:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176822.476997:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176822.477004:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176822.477011:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176822.477012:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176822.477017:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000100:00100000:0.0:1786176822.477022:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235540736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176822.477040:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176822.477045:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176822.477055:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235540736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176822.477075:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176822.477079:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176822.477084:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176822.477088:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176822.477091:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176822.477096:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176822.477101:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11496 00000400:00000200:0.0:1786176822.477107:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176822.477113:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176822.477117:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176822.477120:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.477123:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176822.477126:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.477128:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176822.477130:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176822.477132:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176822.477135:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176822.477136:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176822.477138:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176822.477140:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176822.477146:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.477148:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176822.477153:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.477155:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176822.477158:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176822.477163:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176822.477169:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176822.477175:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176822.477181:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176822.477185:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.477190:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176822.477195:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176822.477202:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.477207:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176822.477215:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786176822.477221:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176822.477226:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176822.477230:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.477236:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176822.477240:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176822.477244:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176822.477252:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176822.477261:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176822.477270:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235540864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176822.477284:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176822.477288:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176822.477294:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235540864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176822.477303:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176822.477306:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176822.477315:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176822.477318:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176822.477320:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176822.477324:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176822.477329:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11496 00000400:00000200:0.0:1786176822.477336:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176822.477342:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176822.477346:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176822.477353:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.477384:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176822.477388:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.477393:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176822.477397:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176822.477400:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176822.477403:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176822.477408:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176822.477411:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176822.477414:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176822.477419:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.477423:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176822.477426:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176822.477427:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176822.477429:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176822.477434:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176822.477437:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176822.477439:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176822.477443:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176822.477447:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.477451:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176822.477454:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176822.477459:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.477462:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176822.477468:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786176822.477471:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176822.477472:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176822.477473:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176822.477474:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176822.477475:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176822.477476:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176822.477479:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176822.477486:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176822.477491:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235540992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176822.477502:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176822.477506:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176822.477510:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235540992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176822.477520:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176822.477526:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11507) 00000800:00000200:1.1:1786176822.477630:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176822.477632:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786176822.477937:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176822.477940:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786176822.478263:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176822.478267:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1786176822.478694:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176822.478696:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000100:00000001:2.0:1786176822.988643:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176822.988654:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176822.988656:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176822.988662:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176822.988666:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176822.988670:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176822.988673:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176822.988675:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176822.988678:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176823.372457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176823.372462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176823.372468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176823.372471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176823.372476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176823.372477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176823.372540:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176823.372544:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176823.372548:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176824.012603:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176824.012618:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176824.012621:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176824.012628:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176824.012633:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176824.012636:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176824.012639:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176824.012642:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176824.012647:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176824.396462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176824.396467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176824.396474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176824.396476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176824.396481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176824.396484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176824.396594:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176824.396600:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176824.396604:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176825.037699:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176825.037715:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176825.037720:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176825.037728:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176825.037734:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176825.037740:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176825.037744:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176825.037748:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176825.037751:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176825.420462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176825.420467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176825.420474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176825.420478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176825.420482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176825.420486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176825.420547:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176825.420551:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176825.420554:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176826.060551:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176826.060561:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176826.060565:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176826.060571:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176826.060576:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176826.060580:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176826.060582:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176826.060584:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176826.060587:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176826.380463:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306173184, at 4306173248 00000100:00000010:0.0:1786176826.380482:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62b40. 00010000:00000001:0.0:1786176826.444436:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176826.444440:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176826.444446:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176826.444449:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176826.444452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176826.444454:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176826.444506:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176826.444510:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176826.444514:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176827.084493:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176827.084500:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176827.084502:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176827.084507:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176827.084510:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176827.084514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176827.084516:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176827.084523:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176827.084526:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176827.468468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176827.468481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176827.468490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176827.468499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176827.468506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176827.468507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176827.468563:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176827.468567:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176827.468569:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176827.596714:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176827.596720:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176827.596725:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176827.596734:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328baa00 x1872930235541120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.596749:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176827.596752:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176827.596756:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176827.596760:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176827.596764:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176827.596770:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235541120:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176827.596777:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176827.596782:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176827.596785:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176827.596788:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176827.596794:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786176827.596798:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176827.596809:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176827.596814:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176827.596822:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176827.596831:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235541120, portal 25 00000100:00000001:2.0:1786176827.596836:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176827.596840:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487992832 : -110371221558784 : ffff9b9e328baa00) 00000100:00000040:2.0:1786176827.596852:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328baa00 x1872930235541120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.596869:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176827.596874:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176827.596879:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176827.596884:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235541120, offset 0 00000400:00000200:2.0:1786176827.596894:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176827.596930:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176827.596941:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176827.596951:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176827.596959:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176827.596968:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176827.596976:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176827.596988:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426294:426294:256:4294967295] 192.168.202.132@tcp LPNI seq info [426256:426256:8:4294967295] 00000400:00000200:2.0:1786176827.596999:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176827.597016:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176827.597028:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176827.597036:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176827.597048:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176827.597054:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786176827.597065:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176827.597071:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:2.0:1786176827.597079:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176827.597082:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176827.597090:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176827.597099:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176827.597104:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176827.597109:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176827.597115:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176827.597124:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328baa00 x1872930235541120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597141:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176827.597148:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328baa00 x1872930235541120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597160:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.597162:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176827.597169:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176827.597177:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176827.597181:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306178304 00000100:00000010:2.0:1786176827.597187:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c6c0. 00000100:00000001:2.0:1786176827.597196:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786176827.597204:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306178304 00000100:00000010:2.0:1786176827.597210:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cba0. 00000100:00000001:2.0:1786176827.597214:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786176827.597219:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176827.597223:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.597226:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176827.597233:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176827/real 1786176827] req@ffff9b9e328baa00 x1872930235541120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176827.597246:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176827.597248:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176827.597251:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176827.597256:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176827.597259:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176827.597263:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176827.597268:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328baa00 x1872930235541120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176827.597283:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328baa00 x1872930235541120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597292:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176827.597295:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597298:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176827.597302:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176827.597308:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328baa00 x1872930235541120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597320:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176827.597328:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328baa00 x1872930235541120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597341:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176827.597346:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597349:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176827.597352:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597355:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176827.597383:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176827.597386:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176827.597389:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176827.597391:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176827.597395:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328baa00 x1872930235541120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176827.597410:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235541120:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176827.597417:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597420:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176827.597425:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328baa00 x1872930235541120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597434:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176827.597437:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176827.597442:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786176827.597447:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176827.597450:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176827.597454:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786176827.597456:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176827.597461:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af000. 02000000:00000001:2.0:1786176827.597464:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176827.597466:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176827.597469:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328baa00. 00000100:00000001:2.0:1786176827.597473:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176827.597477:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597482:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597493:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176827.597499:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.597504:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176827.597507:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176827.597511:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176827.597519:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bad80 x1872930235541248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597534:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176827.597536:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176827.597538:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176827.597540:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176827.597541:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176827.597543:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235541248:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176827.597546:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176827.597548:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176827.597550:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176827.597551:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176827.597554:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786176827.597555:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176827.597558:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176827.597560:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176827.597563:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176827.597565:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235541248, portal 10 00000100:00000001:2.0:1786176827.597567:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176827.597568:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487993728 : -110371221557888 : ffff9b9e328bad80) 00000100:00000040:2.0:1786176827.597573:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bad80 x1872930235541248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597577:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176827.597579:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176827.597580:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176827.597582:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235541248, offset 0 00000400:00000200:2.0:1786176827.597584:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176827.597587:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176827.597590:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176827.597592:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176827.597594:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176827.597595:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176827.597597:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176827.597600:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426295:426295:256:4294967295] 192.168.202.132@tcp LPNI seq info [426257:426257:8:4294967295] 00000400:00000200:2.0:1786176827.597603:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176827.597607:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176827.597610:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176827.597613:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176827.597615:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176827.597617:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786176827.597620:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176827.597622:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:2.0:1786176827.597625:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176827.597626:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176827.597628:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176827.597631:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176827.597633:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176827.597634:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176827.597636:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176827.597638:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bad80 x1872930235541248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597643:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176827.597645:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bad80 x1872930235541248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597650:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.597651:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176827.597653:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176827.597655:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786176827.597657:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176827.597659:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306174468 00000100:00000001:2.0:1786176827.597661:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306178304 00000100:00000001:2.0:1786176827.597662:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306174468 00000100:00000001:2.0:1786176827.597664:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306178304 00000100:00000001:2.0:1786176827.597665:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176827.597667:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.597668:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176827.597670:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176827/real 1786176827] req@ffff9b9e328bad80 x1872930235541248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176827.597676:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176827.597676:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176827.597677:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176827.597679:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176827.597680:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176827.597681:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176827.597683:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bad80 x1872930235541248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176827.597689:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bad80 x1872930235541248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597693:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176827.597694:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597695:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176827.597696:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176827.597698:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bad80 x1872930235541248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597702:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176827.597704:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bad80 x1872930235541248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597708:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176827.597710:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597711:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176827.597712:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597713:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176827.597714:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176827.597716:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176827.597718:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176827.597719:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176827.597722:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bad80 x1872930235541248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176827.597727:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235541248:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176827.597729:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176827.597742:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bbb80 x1872930235541376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597753:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176827.597755:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176827.597757:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176827.597758:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176827.597760:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176827.597763:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235541376:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176827.597767:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176827.597769:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176827.597770:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176827.597773:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176827.597776:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786176827.597779:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176827.597782:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176827.597784:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176827.597787:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176827.597790:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235541376, portal 10 00000100:00000001:2.0:1786176827.597792:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176827.597793:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487997312 : -110371221554304 : ffff9b9e328bbb80) 00000100:00000040:2.0:1786176827.597800:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bbb80 x1872930235541376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597807:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176827.597810:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176827.597813:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176827.597815:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235541376, offset 0 00000400:00000200:2.0:1786176827.597820:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176827.597824:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176827.597828:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176827.597831:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176827.597834:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176827.597835:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176827.597837:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176827.597839:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426296:426296:256:4294967295] 192.168.202.132@tcp LPNI seq info [426258:426258:8:4294967295] 00000400:00000200:2.0:1786176827.597842:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176827.597847:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176827.597850:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176827.597852:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176827.597855:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176827.597857:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786176827.597860:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176827.597861:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:2.0:1786176827.597864:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176827.597865:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176827.597867:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176827.597870:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176827.597872:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176827.597873:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176827.597875:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176827.597878:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bbb80 x1872930235541376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597883:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176827.597885:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bbb80 x1872930235541376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597890:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.597891:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176827.597893:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176827.597896:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786176827.597898:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176827.597899:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306174468 00000100:00000001:2.0:1786176827.597901:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306178304 00000100:00000001:2.0:1786176827.597917:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306174468 00000100:00000001:2.0:1786176827.597918:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306178304 00000100:00000001:2.0:1786176827.597920:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176827.597922:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.597924:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176827.597927:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176827/real 1786176827] req@ffff9b9e328bbb80 x1872930235541376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176827.597932:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176827.597934:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176827.597935:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176827.597937:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176827.597938:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176827.597941:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176827.597944:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bbb80 x1872930235541376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176827.597953:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bbb80 x1872930235541376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597960:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176827.597962:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597963:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176827.597965:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176827.597969:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bbb80 x1872930235541376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597977:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176827.597982:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bbb80 x1872930235541376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.597990:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176827.597992:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597994:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176827.597995:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.597996:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176827.597998:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176827.598000:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176827.598002:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176827.598004:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176827.598006:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bbb80 x1872930235541376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176827.598012:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235541376:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176827.598014:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176827.598016:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8000 x1872930235541504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598021:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176827.598022:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176827.598024:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176827.598025:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176827.598026:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176827.598028:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235541504:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176827.598030:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176827.598031:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176827.598032:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176827.598034:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176827.598037:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786176827.598038:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176827.598041:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176827.598043:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176827.598045:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176827.598048:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235541504, portal 4 00000100:00000001:2.0:1786176827.598049:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176827.598050:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982080 : -110371221569536 : ffff9b9e328b8000) 00000100:00000040:2.0:1786176827.598054:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8000 x1872930235541504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598058:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176827.598060:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176827.598061:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176827.598063:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235541504, offset 0 00000400:00000200:2.0:1786176827.598065:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176827.598068:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176827.598070:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176827.598072:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176827.598074:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176827.598076:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176827.598077:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176827.598079:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426297:426297:256:4294967295] 192.168.202.132@tcp LPNI seq info [426259:426259:8:4294967295] 00000400:00000200:2.0:1786176827.598082:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176827.598086:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176827.598089:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176827.598091:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176827.598094:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176827.598096:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786176827.598099:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176827.598101:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:2.0:1786176827.598103:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176827.598104:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176827.598107:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176827.598109:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176827.598111:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176827.598112:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176827.598114:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176827.598116:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8000 x1872930235541504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598121:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176827.598123:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8000 x1872930235541504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598127:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.598129:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176827.598130:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176827.598132:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786176827.598134:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176827.598136:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306174468 00000100:00000001:2.0:1786176827.598137:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306178304 00000100:00000001:2.0:1786176827.598139:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306174468 00000100:00000001:2.0:1786176827.598140:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306178304 00000100:00000001:2.0:1786176827.598141:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176827.598142:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.598144:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176827.598146:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176827/real 1786176827] req@ffff9b9e328b8000 x1872930235541504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176827.598151:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176827.598151:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176827.598153:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176827.598154:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176827.598155:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176827.598156:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176827.598158:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8000 x1872930235541504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176827.598164:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8000 x1872930235541504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598168:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176827.598169:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598170:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176827.598171:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176827.598174:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8000 x1872930235541504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598177:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176827.598180:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8000 x1872930235541504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598184:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176827.598185:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598186:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176827.598187:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598188:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176827.598189:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176827.598190:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176827.598192:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176827.598193:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176827.598195:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8000 x1872930235541504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176827.598200:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235541504:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176827.598202:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598204:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176827.598206:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bad80 x1872930235541248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598210:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176827.598211:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176827.598213:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786176827.598214:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176827.598215:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176827.598217:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176827.598218:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176827.598219:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261adc00. 02000000:00000001:2.0:1786176827.598221:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176827.598222:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176827.598223:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bad80. 00000100:00000001:2.0:1786176827.598225:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176827.598226:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598227:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176827.598229:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bbb80 x1872930235541376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598234:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176827.598234:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176827.598235:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786176827.598236:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176827.598237:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176827.598238:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176827.598239:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176827.598240:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:2.0:1786176827.598241:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176827.598242:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176827.598242:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bbb80. 00000100:00000001:2.0:1786176827.598244:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176827.598244:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598245:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176827.598247:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8000 x1872930235541504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598251:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176827.598251:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176827.598252:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786176827.598254:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176827.598254:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176827.598255:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176827.598257:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176827.598258:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:2.0:1786176827.598259:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176827.598259:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176827.598260:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8000. 00000100:00000001:2.0:1786176827.598262:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176827.598262:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598264:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598266:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176827.598267:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.598269:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176827.598270:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176827.598272:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176827.598275:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9500 x1872930235541632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598282:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176827.598284:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176827.598286:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176827.598287:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176827.598289:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176827.598292:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235541632:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176827.598296:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176827.598297:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176827.598299:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176827.598301:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176827.598304:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786176827.598306:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176827.598309:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176827.598312:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176827.598315:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176827.598318:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235541632, portal 4 00000100:00000001:2.0:1786176827.598320:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176827.598322:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487987456 : -110371221564160 : ffff9b9e328b9500) 00000100:00000040:2.0:1786176827.598328:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9500 x1872930235541632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598342:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176827.598344:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176827.598347:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176827.598350:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235541632, offset 0 00000400:00000200:2.0:1786176827.598354:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176827.598379:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176827.598385:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176827.598388:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176827.598391:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176827.598393:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176827.598396:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176827.598399:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426298:426298:256:4294967295] 192.168.202.132@tcp LPNI seq info [426260:426260:8:4294967295] 00000400:00000200:2.0:1786176827.598404:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176827.598411:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176827.598416:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176827.598420:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176827.598424:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176827.598427:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1786176827.598432:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176827.598435:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:2.0:1786176827.598439:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176827.598441:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176827.598445:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176827.598448:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176827.598451:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176827.598454:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176827.598456:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176827.598460:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9500 x1872930235541632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598469:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176827.598472:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9500 x1872930235541632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176852 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598486:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.598487:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176827.598491:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176827.598494:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786176827.598498:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176827.598500:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306174471 00000100:00000001:2.0:1786176827.598503:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306178304 00000100:00000001:2.0:1786176827.598505:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306174471 00000100:00000001:2.0:1786176827.598508:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306178304 00000100:00000001:2.0:1786176827.598510:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176827.598512:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.598514:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176827.598518:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176827/real 1786176827] req@ffff9b9e328b9500 x1872930235541632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176827.598527:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176827.598528:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176827.598530:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176827.598533:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176827.598534:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176827.598537:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176827.598540:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9500 x1872930235541632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176827.598550:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9500 x1872930235541632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598558:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176827.598559:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598561:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176827.598562:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176827.598566:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9500 x1872930235541632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598574:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176827.598578:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9500 x1872930235541632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598586:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176827.598588:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598590:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176827.598591:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598593:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176827.598595:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176827.598598:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176827.598601:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176827.598604:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176827.598608:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9500 x1872930235541632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176827.598616:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235541632:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176827.598621:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598623:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176827.598626:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9500 x1872930235541632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176852 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176827.598633:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176827.598635:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176827.598637:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786176827.598639:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176827.598641:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176827.598642:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176827.598645:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176827.598647:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:2.0:1786176827.598649:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176827.598651:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176827.598653:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9500. 00000100:00000001:2.0:1786176827.598655:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176827.598657:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598659:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176827.598662:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176827.598664:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176827.598666:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176827.598667:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176828.108497:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176828.108506:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176828.108509:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176828.108515:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176828.108520:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176828.108525:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176828.108528:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176828.108531:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176828.108535:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176828.492472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176828.492490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176828.492503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176828.492506:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176828.492509:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176828.492513:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176828.492517:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176828.492520:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176828.492523:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176828.492526:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176828.492531:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176828.492532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176828.492587:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176828.492592:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176828.492596:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176829.132466:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176829.132473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176829.132475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176829.132480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176829.132484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176829.132488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176829.132490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176829.132492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176829.132495:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176829.516463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176829.516468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176829.516479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176829.516483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176829.516489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176829.516493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176829.516564:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176829.516569:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176829.516572:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176830.156439:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176830.156446:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176830.156448:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176830.156453:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176830.156456:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176830.156460:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176830.156463:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176830.156465:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176830.156468:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176830.540442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176830.540446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176830.540455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176830.540459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176830.540466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176830.540471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176830.540568:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176830.540572:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176830.540575:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176831.180473:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176831.180480:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176831.180482:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176831.180486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176831.180490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176831.180493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176831.180496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176831.180499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176831.180502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176831.500456:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306178304, at 4306178368 00000100:00000010:0.0:1786176831.500469:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c6c0. 00010000:00000001:0.0:1786176831.564444:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176831.564450:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176831.564454:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176831.564457:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176831.564471:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176831.564474:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176831.564479:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176831.564481:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176831.564485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176831.564487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176831.564489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176831.564491:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176831.564491:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176831.564493:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176831.564494:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176831.564496:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176831.564497:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176831.564499:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176831.564555:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176831.564558:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176831.564561:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176832.204437:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176832.204445:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176832.204447:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176832.204452:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176832.204455:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176832.204459:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176832.204461:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176832.204463:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176832.204465:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176832.588438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176832.588442:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176832.588449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176832.588451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176832.588454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176832.588455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176832.588515:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176832.588520:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176832.588523:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176832.716656:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176832.716661:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176832.716665:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176832.716674:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9880 x1872930235541760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.716693:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176832.716697:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176832.716704:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176832.716708:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176832.716711:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176832.716718:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235541760:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176832.716723:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176832.716728:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176832.716733:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176832.716740:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176832.716750:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786176832.716755:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176832.716768:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176832.716775:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176832.716786:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176832.716797:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235541760, portal 25 00000100:00000001:2.0:1786176832.716803:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176832.716806:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487988352 : -110371221563264 : ffff9b9e328b9880) 00000100:00000040:2.0:1786176832.716816:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9880 x1872930235541760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.716834:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176832.716842:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176832.716846:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176832.716866:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235541760, offset 0 00000400:00000200:2.0:1786176832.716880:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176832.716894:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176832.716908:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176832.716916:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176832.716924:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176832.716933:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176832.716940:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176832.716949:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426299:426299:256:4294967295] 192.168.202.132@tcp LPNI seq info [426261:426261:8:4294967295] 00000400:00000200:2.0:1786176832.716961:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176832.716978:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176832.716996:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176832.717007:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176832.717028:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176832.717034:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176832.717045:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176832.717050:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1786176832.717059:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176832.717064:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176832.717073:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176832.717083:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176832.717091:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176832.717097:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176832.717106:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176832.717116:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9880 x1872930235541760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717133:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176832.717141:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9880 x1872930235541760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717156:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.717160:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176832.717167:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176832.717177:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176832.717182:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306183424 00000100:00000010:2.0:1786176832.717190:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c840. 00000100:00000001:2.0:1786176832.717200:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786176832.717211:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306183424 00000100:00000010:2.0:1786176832.717216:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c240. 00000100:00000001:2.0:1786176832.717220:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786176832.717224:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176832.717228:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.717233:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176832.717241:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176832/real 1786176832] req@ffff9b9e328b9880 x1872930235541760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176832.717263:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176832.717267:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176832.717271:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176832.717278:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176832.717284:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176832.717293:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176832.717303:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9880 x1872930235541760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176832.717329:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9880 x1872930235541760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717340:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176832.717343:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717348:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176832.717351:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176832.717379:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9880 x1872930235541760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717387:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176832.717396:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9880 x1872930235541760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717406:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176832.717411:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717417:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176832.717420:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717426:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176832.717431:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176832.717440:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176832.717448:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176832.717453:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176832.717459:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9880 x1872930235541760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176832.717468:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235541760:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176832.717473:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717480:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176832.717482:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9880 x1872930235541760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717487:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176832.717489:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176832.717491:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786176832.717494:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176832.717495:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176832.717496:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786176832.717498:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176832.717500:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:2.0:1786176832.717503:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176832.717504:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176832.717505:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9880. 00000100:00000001:2.0:1786176832.717507:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176832.717509:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717510:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717516:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176832.717517:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.717519:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176832.717520:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176832.717521:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176832.717524:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8380 x1872930235541888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717529:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176832.717530:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176832.717532:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176832.717533:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176832.717534:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176832.717536:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235541888:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176832.717538:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176832.717540:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176832.717541:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176832.717542:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176832.717544:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786176832.717546:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176832.717549:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176832.717551:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176832.717554:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176832.717556:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235541888, portal 10 00000100:00000001:2.0:1786176832.717558:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176832.717559:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982976 : -110371221568640 : ffff9b9e328b8380) 00000100:00000040:2.0:1786176832.717564:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8380 x1872930235541888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717568:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176832.717570:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176832.717571:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176832.717573:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235541888, offset 0 00000400:00000200:2.0:1786176832.717575:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176832.717578:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176832.717581:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176832.717584:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176832.717586:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176832.717588:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176832.717589:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176832.717592:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426300:426300:256:4294967295] 192.168.202.132@tcp LPNI seq info [426262:426262:8:4294967295] 00000400:00000200:2.0:1786176832.717594:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176832.717599:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176832.717602:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176832.717605:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176832.717608:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176832.717610:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176832.717613:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176832.717615:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1786176832.717618:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176832.717619:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176832.717621:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176832.717624:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176832.717626:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176832.717627:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176832.717629:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176832.717631:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8380 x1872930235541888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717637:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176832.717639:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8380 x1872930235541888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717644:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.717645:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176832.717647:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176832.717649:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786176832.717651:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176832.717653:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306179588 00000100:00000001:2.0:1786176832.717655:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306183424 00000100:00000001:2.0:1786176832.717657:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306179588 00000100:00000001:2.0:1786176832.717658:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306183424 00000100:00000001:2.0:1786176832.717659:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176832.717661:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.717662:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176832.717664:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176832/real 1786176832] req@ffff9b9e328b8380 x1872930235541888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176832.717669:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176832.717670:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176832.717671:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176832.717672:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176832.717674:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176832.717675:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176832.717677:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8380 x1872930235541888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176832.717682:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8380 x1872930235541888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717686:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176832.717687:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717688:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176832.717689:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176832.717691:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8380 x1872930235541888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717695:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176832.717697:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8380 x1872930235541888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717702:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176832.717703:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717705:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176832.717706:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717707:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176832.717709:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176832.717711:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176832.717713:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176832.717714:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176832.717716:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8380 x1872930235541888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176832.717721:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235541888:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176832.717723:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176832.717726:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9500 x1872930235542016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717730:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176832.717731:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176832.717732:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176832.717733:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176832.717735:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176832.717738:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235542016:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176832.717742:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176832.717743:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176832.717745:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176832.717747:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176832.717750:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786176832.717752:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176832.717755:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176832.717758:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176832.717761:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176832.717764:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235542016, portal 10 00000100:00000001:2.0:1786176832.717766:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176832.717768:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487987456 : -110371221564160 : ffff9b9e328b9500) 00000100:00000040:2.0:1786176832.717775:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9500 x1872930235542016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717783:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176832.717785:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176832.717788:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176832.717790:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235542016, offset 0 00000400:00000200:2.0:1786176832.717794:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176832.717798:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176832.717800:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176832.717802:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176832.717804:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176832.717806:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176832.717807:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176832.717810:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426301:426301:256:4294967295] 192.168.202.132@tcp LPNI seq info [426263:426263:8:4294967295] 00000400:00000200:2.0:1786176832.717813:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176832.717817:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176832.717821:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176832.717823:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176832.717825:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176832.717827:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176832.717830:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176832.717833:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1786176832.717835:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176832.717836:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176832.717839:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176832.717841:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176832.717843:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176832.717844:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176832.717846:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176832.717848:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9500 x1872930235542016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717871:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176832.717874:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9500 x1872930235542016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717878:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.717879:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176832.717881:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176832.717884:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786176832.717886:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176832.717887:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306179588 00000100:00000001:2.0:1786176832.717890:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306183424 00000100:00000001:2.0:1786176832.717892:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306179588 00000100:00000001:2.0:1786176832.717895:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306183424 00000100:00000001:2.0:1786176832.717897:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176832.717899:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.717901:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176832.717905:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176832/real 1786176832] req@ffff9b9e328b9500 x1872930235542016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176832.717913:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176832.717915:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176832.717917:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176832.717919:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176832.717921:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176832.717923:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176832.717927:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9500 x1872930235542016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176832.717937:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9500 x1872930235542016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717944:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176832.717945:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717947:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176832.717948:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176832.717950:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9500 x1872930235542016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717955:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176832.717958:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9500 x1872930235542016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717962:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176832.717964:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717964:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176832.717965:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.717966:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176832.717968:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176832.717969:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176832.717971:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176832.717972:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176832.717975:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9500 x1872930235542016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176832.717979:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235542016:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176832.717982:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176832.717986:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8000 x1872930235542144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.717993:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176832.717994:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176832.717996:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176832.717998:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176832.718000:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176832.718004:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235542144:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176832.718007:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176832.718009:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176832.718011:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176832.718013:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176832.718017:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786176832.718019:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176832.718023:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176832.718026:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176832.718030:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176832.718034:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235542144, portal 4 00000100:00000001:2.0:1786176832.718035:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176832.718036:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982080 : -110371221569536 : ffff9b9e328b8000) 00000100:00000040:2.0:1786176832.718041:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8000 x1872930235542144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718045:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176832.718047:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176832.718048:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176832.718050:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235542144, offset 0 00000400:00000200:2.0:1786176832.718053:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176832.718055:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176832.718058:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176832.718060:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176832.718062:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176832.718064:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176832.718066:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176832.718068:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426302:426302:256:4294967295] 192.168.202.132@tcp LPNI seq info [426264:426264:8:4294967295] 00000400:00000200:2.0:1786176832.718071:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176832.718075:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176832.718078:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176832.718080:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176832.718083:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176832.718085:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176832.718088:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176832.718090:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1786176832.718094:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176832.718096:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176832.718099:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176832.718103:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176832.718105:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176832.718107:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176832.718110:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176832.718113:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8000 x1872930235542144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718122:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176832.718125:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8000 x1872930235542144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718133:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.718135:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176832.718137:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176832.718140:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786176832.718144:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176832.718145:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306179588 00000100:00000001:2.0:1786176832.718147:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306183424 00000100:00000001:2.0:1786176832.718149:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306179588 00000100:00000001:2.0:1786176832.718150:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306183424 00000100:00000001:2.0:1786176832.718152:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176832.718153:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.718154:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176832.718157:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176832/real 1786176832] req@ffff9b9e328b8000 x1872930235542144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176832.718162:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176832.718164:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176832.718165:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176832.718166:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176832.718167:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176832.718169:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176832.718171:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8000 x1872930235542144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176832.718176:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8000 x1872930235542144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718181:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176832.718181:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718183:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176832.718183:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176832.718186:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8000 x1872930235542144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718190:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176832.718192:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8000 x1872930235542144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718196:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176832.718197:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718198:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176832.718199:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718200:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176832.718201:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176832.718203:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176832.718205:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176832.718206:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176832.718208:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8000 x1872930235542144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176832.718218:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235542144:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176832.718220:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176832.718223:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bbb80 x1872930235542272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718227:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176832.718228:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176832.718229:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176832.718230:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176832.718231:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176832.718233:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235542272:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176832.718235:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176832.718236:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176832.718237:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176832.718238:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176832.718241:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa400. 02000000:00000001:2.0:1786176832.718242:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176832.718244:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176832.718246:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176832.718248:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176832.718250:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235542272, portal 4 00000100:00000001:2.0:1786176832.718251:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176832.718252:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487997312 : -110371221554304 : ffff9b9e328bbb80) 00000100:00000040:2.0:1786176832.718256:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bbb80 x1872930235542272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718261:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176832.718262:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176832.718264:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176832.718265:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235542272, offset 0 00000400:00000200:2.0:1786176832.718268:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176832.718270:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176832.718273:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176832.718276:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176832.718279:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176832.718281:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176832.718284:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176832.718287:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426303:426303:256:4294967295] 192.168.202.132@tcp LPNI seq info [426265:426265:8:4294967295] 00000400:00000200:2.0:1786176832.718291:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176832.718298:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176832.718303:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176832.718306:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176832.718311:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176832.718313:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1786176832.718318:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176832.718321:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:2.0:1786176832.718325:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176832.718327:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176832.718330:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176832.718332:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176832.718334:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176832.718336:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176832.718337:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176832.718340:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bbb80 x1872930235542272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718345:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176832.718348:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bbb80 x1872930235542272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176857 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718352:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.718353:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176832.718355:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176832.718373:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786176832.718376:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176832.718378:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306179591 00000100:00000001:2.0:1786176832.718379:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306183424 00000100:00000001:2.0:1786176832.718381:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306179591 00000100:00000001:2.0:1786176832.718383:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306183424 00000100:00000001:2.0:1786176832.718384:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176832.718386:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.718387:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176832.718389:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176832/real 1786176832] req@ffff9b9e328bbb80 x1872930235542272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176832.718394:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176832.718395:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176832.718396:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176832.718398:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176832.718399:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176832.718401:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176832.718403:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bbb80 x1872930235542272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176832.718408:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bbb80 x1872930235542272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718413:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176832.718414:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718416:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176832.718417:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176832.718421:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bbb80 x1872930235542272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718428:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176832.718432:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bbb80 x1872930235542272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718439:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176832.718442:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718444:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176832.718445:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718447:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176832.718449:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176832.718452:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176832.718456:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176832.718458:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176832.718462:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bbb80 x1872930235542272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176832.718470:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235542272:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176832.718472:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718474:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176832.718476:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8380 x1872930235541888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718480:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176832.718482:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176832.718483:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786176832.718485:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176832.718486:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176832.718487:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176832.718488:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176832.718490:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:2.0:1786176832.718491:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176832.718492:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176832.718493:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8380. 00000100:00000001:2.0:1786176832.718495:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176832.718496:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718497:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176832.718499:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9500 x1872930235542016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718508:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176832.718509:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176832.718510:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786176832.718512:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176832.718512:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176832.718513:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176832.718514:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176832.718515:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:2.0:1786176832.718516:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176832.718517:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176832.718518:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9500. 00000100:00000001:2.0:1786176832.718519:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176832.718520:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718521:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176832.718522:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8000 x1872930235542144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718526:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176832.718527:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176832.718528:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786176832.718529:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176832.718530:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176832.718530:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176832.718531:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176832.718532:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:2.0:1786176832.718533:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176832.718534:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176832.718535:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8000. 00000100:00000001:2.0:1786176832.718536:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176832.718537:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718538:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176832.718540:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bbb80 x1872930235542272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176857 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176832.718543:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176832.718544:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176832.718545:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:2.0:1786176832.718547:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176832.718548:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176832.718548:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176832.718550:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176832.718551:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:2.0:1786176832.718552:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176832.718552:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176832.718553:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bbb80. 00000100:00000001:2.0:1786176832.718555:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176832.718556:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718557:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176832.718560:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176832.718560:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176832.718562:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176832.718563:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176833.228468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176833.228487:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176833.228489:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176833.228500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176833.228505:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176833.228509:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176833.228513:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176833.228516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176833.228520:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176833.612455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176833.612458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176833.612464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176833.612466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176833.612469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176833.612470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176833.612525:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176833.612529:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176833.612532:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176834.252457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176834.252466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176834.252468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176834.252475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176834.252479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176834.252483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176834.252486:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176834.252489:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176834.252492:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176834.636453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176834.636460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176834.636474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176834.636481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176834.636487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176834.636490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176834.636584:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176834.636588:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176834.636592:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176835.276529:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176835.276540:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176835.276543:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176835.276551:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176835.276556:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176835.276560:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176835.276562:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176835.276566:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176835.276568:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176835.660512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176835.660522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176835.660530:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176835.660535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176835.660540:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176835.660544:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176835.660613:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176835.660617:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176835.660620:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176836.300426:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176836.300433:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176836.300435:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176836.300439:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176836.300443:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176836.300446:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176836.300448:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176836.300451:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176836.300461:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176836.620494:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306183424, at 4306183488 00000100:00000010:0.0:1786176836.620514:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c840. 00010000:00000001:0.0:1786176836.684490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176836.684495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176836.684503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176836.684506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176836.684510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176836.684512:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176836.684571:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176836.684575:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176836.684579:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176837.324467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176837.324474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176837.324476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176837.324481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176837.324486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176837.324489:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176837.324493:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176837.324496:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176837.324507:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176837.708432:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176837.708438:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176837.708451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176837.708454:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176837.708460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176837.708463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176837.708523:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176837.708526:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176837.708529:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786176837.836467:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176837.836484:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786176837.836487:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176837.836492:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176837.836496:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176837.836504:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11512 00000400:00000200:2.0:1786176837.836513:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176837.836519:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176837.836522:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176837.836524:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836527:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176837.836529:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836531:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176837.836534:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176837.836536:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786176837.836538:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176837.836541:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786176837.836543:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786176837.836546:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786176837.836548:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836550:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176837.836552:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836554:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176837.836558:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786176837.836561:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836564:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176837.836567:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176837.836570:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176837.836574:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.836576:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176837.836579:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176837.836589:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786176837.836593:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786176837.836597:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176837.836599:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.836602:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176837.836609:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176837.836613:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176837.836620:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:2.0:1786176837.836626:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176837.836633:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235542400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176837.836642:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176837.836645:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176837.836649:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235542400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176837.836666:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176837.836669:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176837.836671:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786176837.836673:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176837.836674:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176837.836677:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176837.836678:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11512 00000400:00000200:2.0:1786176837.836682:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176837.836685:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176837.836688:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176837.836691:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836695:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176837.836697:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836700:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176837.836705:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176837.836707:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786176837.836711:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176837.836716:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786176837.836720:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786176837.836724:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786176837.836734:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836738:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176837.836744:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836748:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176837.836751:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786176837.836755:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836757:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176837.836760:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176837.836765:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176837.836769:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.836772:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176837.836777:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176837.836782:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.836787:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176837.836790:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 00000020:00000040:2.0:1786176837.836794:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176837.836799:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176837.836801:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.836821:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176837.836822:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176837.836823:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176837.836827:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786176837.836830:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176837.836835:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235542528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176837.836842:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176837.836845:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176837.836857:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235542528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176837.836873:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176837.836878:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176837.836884:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786176837.836888:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176837.836890:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176837.836898:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176837.836904:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11512 00000400:00000200:2.0:1786176837.836912:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176837.836919:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176837.836923:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176837.836936:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836939:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176837.836942:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836944:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176837.836949:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176837.836952:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786176837.836955:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176837.836959:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786176837.836960:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786176837.836962:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786176837.836968:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836972:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176837.836973:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836975:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176837.836977:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786176837.836981:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176837.836986:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176837.836987:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176837.836990:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176837.836993:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.836997:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176837.837000:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176837.837003:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.837008:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176837.837012:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786176837.837017:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176837.837023:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176837.837027:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.837032:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176837.837035:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176837.837039:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176837.837048:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab400. 00000100:00000001:2.0:1786176837.837054:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176837.837065:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235542656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176837.837089:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176837.837093:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176837.837099:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:2.0:1786176837.837101:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235542656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176837.837109:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176837.837111:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176837.837114:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786176837.837115:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176837.837116:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176837.837118:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176837.837120:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11512 00000400:00000200:2.0:1786176837.837126:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176837.837131:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176837.837134:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176837.837138:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.837144:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176837.837145:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.837147:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176837.837149:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176837.837150:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786176837.837152:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176837.837154:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786176837.837160:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786176837.837162:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786176837.837165:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.837170:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176837.837172:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.837174:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176837.837176:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786176837.837181:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176837.837185:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176837.837191:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176837.837194:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176837.837198:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.837200:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176837.837203:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176837.837207:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.837210:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176837.837219:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000020:00000040:2.0:1786176837.837223:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176837.837226:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176837.837227:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.837228:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176837.837230:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176837.837234:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176837.837242:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:2.0:1786176837.837247:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176837.837255:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235542784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176837.837274:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176837.837279:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176837.837289:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235542784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176837.837310:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176837.837314:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176837.837319:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786176837.837324:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176837.837328:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176837.837335:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176837.837341:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11512 00000400:00000200:2.0:1786176837.837350:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176837.837392:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176837.837397:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176837.837404:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.837408:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176837.837414:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.837419:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176837.837424:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176837.837426:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786176837.837429:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176837.837433:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786176837.837438:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786176837.837442:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786176837.837450:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.837456:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176837.837461:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176837.837466:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176837.837472:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786176837.837480:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176837.837486:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176837.837491:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176837.837496:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176837.837502:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.837507:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176837.837511:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176837.837517:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.837521:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176837.837527:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734700. 00000020:00000040:2.0:1786176837.837533:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176837.837539:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176837.837542:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176837.837547:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176837.837550:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176837.837553:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176837.837562:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:2.0:1786176837.837569:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176837.837579:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734700 x1872930235542912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176837.837595:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176837.837599:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176837.837606:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:2.0:1786176837.837608:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088313762] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734700 x1872930235542912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176837.837624:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176837.837633:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11522) 00000800:00000200:1.1:1786176837.837782:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176837.837961:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176837.838101:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786176838.348579:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176838.348585:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176838.348587:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176838.348591:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176838.348594:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176838.348597:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176838.348599:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176838.348600:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176838.348602:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176838.732496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176838.732501:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176838.732511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176838.732516:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176838.732521:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176838.732526:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176838.732531:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176838.732535:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176838.732540:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176838.732545:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176838.732549:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176838.732551:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176838.732606:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176838.732610:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176838.732613:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176839.372463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176839.372470:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176839.372472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176839.372477:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176839.372481:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176839.372485:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176839.372488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176839.372490:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176839.372493:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176839.756454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176839.756460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176839.756466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176839.756468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176839.756472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176839.756474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176839.756526:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176839.756529:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176839.756532:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176840.396420:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176840.396427:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176840.396429:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176840.396434:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176840.396437:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176840.396440:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176840.396442:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176840.396444:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176840.396448:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176840.780498:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176840.780505:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176840.780511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176840.780513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176840.780518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176840.780519:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176840.780587:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176840.780592:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176840.780595:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1786176840.908614:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:0.0:1786176840.908651:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1786176840.908659:0:2382:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176840.908662:0:2382:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:0.0:1786176840.908666:0:2382:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176840.908670:0:2382:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1786176840.908674:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176840.908677:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:0.0:1786176840.908682:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176840.908685:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176840.908690:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e088 00000400:00000010:0.0:1786176840.908692:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e088. 00000100:00000001:0.0:1786176840.908696:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176840.908702:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735c00 x1872930235542400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176840.908723:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176840.908726:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235542400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176840.908732:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176840.908733:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176840.908741:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176840.908742:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:0.0:1786176840.908746:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176840.908748:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176840.908750:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e908 00000400:00000010:0.0:1786176840.908752:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e908. 00000100:00000001:0.0:1786176840.908754:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176840.908757:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31737b80 x1872930235542528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176840.908764:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176840.908766:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235542528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176840.908770:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176840.908771:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176840.908789:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176840.908793:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:0.0:1786176840.908800:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176840.908807:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176840.908814:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ea18 00000400:00000010:0.0:1786176840.908816:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ea18. 00000100:00000001:0.0:1786176840.908821:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176840.908831:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735500 x1872930235542656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176840.908846:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176840.908853:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235542656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176840.908868:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176840.908874:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176840.908886:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176840.908893:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:0.0:1786176840.908899:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176840.908909:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176840.908917:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e880 00000400:00000010:0.0:1786176840.908924:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e880. 00000100:00000001:0.0:1786176840.908929:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176840.908938:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735880 x1872930235542784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176840.908956:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176840.908964:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735880 x1872930235542784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176840.908980:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176840.908984:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176840.908996:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176840.909002:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:0.0:1786176840.909007:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176840.909016:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176840.909023:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ec38 00000400:00000010:0.0:1786176840.909028:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ec38. 00000100:00000001:0.0:1786176840.909033:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176840.909042:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31734700 x1872930235542912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176862 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176840.909057:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176840.909066:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734700 x1872930235542912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176862 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176840.909079:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176840.909084:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176841.420469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176841.420476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176841.420478:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176841.420482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176841.420486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176841.420489:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176841.420492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176841.420494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176841.420497:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176841.740453:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306188544, at 4306188608 00000100:00000010:0.0:1786176841.740472:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62660. 00010000:00000001:0.0:1786176841.804455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176841.804460:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176841.804462:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176841.804466:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176841.804469:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176841.804471:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176841.804474:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176841.804477:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176841.804481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176841.804482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176841.804486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176841.804488:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176841.804489:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176841.804491:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176841.804492:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176841.804494:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176841.804495:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176841.804497:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176841.804562:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176841.804567:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176841.804570:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176842.444444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176842.444452:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176842.444454:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176842.444461:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176842.444467:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176842.444472:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176842.444476:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176842.444479:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176842.444483:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176842.828488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176842.828493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176842.828504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176842.828507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176842.828513:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176842.828516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176842.828581:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176842.828584:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176842.828587:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176842.956536:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176842.956545:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176842.956547:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176842.956550:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176842.956553:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176842.956559:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11517 00000400:00000200:0.0:1786176842.956567:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176842.956572:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176842.956575:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176842.956577:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956581:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176842.956584:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956586:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176842.956591:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176842.956593:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176842.956596:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176842.956599:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176842.956602:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176842.956605:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176842.956608:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956611:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176842.956613:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956615:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176842.956617:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176842.956621:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956623:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176842.956626:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176842.956630:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176842.956632:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.956634:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176842.956637:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176842.956643:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786176842.956647:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176842.956652:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176842.956654:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.956657:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176842.956660:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176842.956664:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176842.956673:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176842.956680:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176842.956690:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235543040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176842.956698:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176842.956702:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176842.956705:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235543040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176842.956736:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176842.956740:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176842.956744:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176842.956746:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176842.956747:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176842.956750:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176842.956753:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11517 00000400:00000200:0.0:1786176842.956771:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176842.956774:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176842.956776:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176842.956778:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956781:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176842.956783:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956785:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176842.956787:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176842.956788:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176842.956790:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176842.956792:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176842.956794:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176842.956796:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176842.956800:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956801:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176842.956803:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956805:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176842.956807:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176842.956809:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956812:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176842.956813:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176842.956818:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176842.956821:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.956823:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176842.956827:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176842.956832:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.956834:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176842.956838:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786176842.956841:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176842.956844:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176842.956845:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.956847:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176842.956849:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176842.956851:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176842.956855:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176842.956860:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176842.956865:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235543168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176842.956873:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176842.956876:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176842.956880:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235543168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176842.956889:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176842.956892:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176842.956896:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176842.956897:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176842.956899:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176842.956902:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176842.956905:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11517 00000400:00000200:0.0:1786176842.956909:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176842.956912:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176842.956915:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176842.956917:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956919:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176842.956921:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956923:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176842.956926:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176842.956927:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176842.956929:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176842.956930:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176842.956932:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176842.956934:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176842.956938:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956940:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176842.956941:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956943:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176842.956945:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176842.956948:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176842.956950:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176842.956952:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176842.956955:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176842.956957:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.956959:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176842.956961:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176842.956963:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.956965:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176842.956968:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786176842.956970:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176842.956972:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176842.956974:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.956975:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176842.956977:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176842.956978:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176842.956982:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176842.956985:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176842.956989:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235543296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176842.957001:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176842.957003:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176842.957007:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235543296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176842.957014:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176842.957017:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176842.957020:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176842.957022:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176842.957023:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176842.957026:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176842.957028:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11517 00000400:00000200:0.0:1786176842.957032:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176842.957035:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176842.957037:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176842.957039:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.957041:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176842.957043:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.957045:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176842.957047:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176842.957049:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176842.957050:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176842.957052:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176842.957054:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176842.957057:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176842.957060:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.957062:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176842.957064:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.957066:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176842.957069:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176842.957072:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176842.957075:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176842.957077:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176842.957079:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176842.957082:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.957085:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176842.957088:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176842.957091:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.957093:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176842.957097:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786176842.957100:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176842.957102:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176842.957103:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.957105:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176842.957107:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176842.957109:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176842.957113:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176842.957117:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176842.957122:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235543424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176842.957131:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176842.957133:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176842.957137:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235543424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176842.957146:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176842.957149:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176842.957152:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176842.957154:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176842.957155:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176842.957158:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176842.957161:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11517 00000400:00000200:0.0:1786176842.957165:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176842.957168:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176842.957170:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176842.957173:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.957175:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176842.957178:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.957181:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176842.957183:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176842.957185:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176842.957186:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176842.957189:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176842.957191:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176842.957193:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176842.957195:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.957196:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176842.957198:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176842.957199:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176842.957202:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176842.957205:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176842.957207:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176842.957209:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176842.957212:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176842.957214:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.957216:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176842.957218:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176842.957221:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.957223:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176842.957226:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786176842.957229:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176842.957232:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176842.957234:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176842.957236:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176842.957237:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176842.957239:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176842.957243:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176842.957246:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176842.957251:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235543552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176842.957258:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176842.957260:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176842.957264:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235543552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176842.957272:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176842.957277:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11527) 00000800:00000200:1.1:1786176842.957562:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176842.957566:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176842.958417:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176842.958419:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176842.958716:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176842.958719:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176842.959048:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176842.959051:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176842.959355:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176842.959387:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1786176843.468468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176843.468476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176843.468479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176843.468485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176843.468490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176843.468495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176843.468497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176843.468500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176843.468503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176843.852450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176843.852456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176843.852464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176843.852468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176843.852476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176843.852480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176843.852581:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176843.852589:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176843.852596:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176844.492441:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176844.492449:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176844.492451:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176844.492456:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176844.492459:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176844.492464:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176844.492467:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176844.492469:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176844.492472:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176844.876514:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176844.876522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176844.876532:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176844.876537:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176844.876545:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176844.876550:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176844.876618:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176844.876623:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176844.876626:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176845.516487:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176845.516497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176845.516500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176845.516507:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176845.516511:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176845.516516:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176845.516519:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176845.516523:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176845.516526:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176845.900505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176845.900512:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176845.900518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176845.900520:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176845.900533:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176845.900534:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176845.903831:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176845.903838:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176845.903844:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176846.540495:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176846.540504:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176846.540506:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176846.540512:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176846.540516:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176846.540520:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176846.540523:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176846.540525:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176846.540528:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176846.860458:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306193665, at 4306193728 00000100:00000010:0.0:1786176846.860472:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62a80. 00010000:00000001:0.0:1786176846.924524:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176846.924530:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176846.924537:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176846.924540:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176846.924545:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176846.924548:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176846.924628:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176846.924632:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176846.924635:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176847.564557:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176847.564564:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176847.564568:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176847.564574:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176847.564579:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176847.564584:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176847.564589:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176847.564591:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176847.564595:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176847.948442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176847.948447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176847.948452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176847.948455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176847.948460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176847.948462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176847.948521:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176847.948525:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176847.948528:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176848.076463:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176848.076472:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176848.076475:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176848.076478:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176848.076481:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176848.076487:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11522 00000400:00000200:0.0:1786176848.076495:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176848.076500:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176848.076502:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176848.076505:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076507:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176848.076509:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076511:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176848.076515:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176848.076516:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176848.076518:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176848.076521:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176848.076522:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176848.076525:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176848.076527:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076529:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176848.076530:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076532:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176848.076534:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176848.076537:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076539:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176848.076541:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176848.076544:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176848.076547:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.076549:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176848.076551:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176848.076561:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176848.076565:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176848.076569:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176848.076571:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.076574:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176848.076577:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176848.076581:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176848.076590:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176848.076596:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176848.076606:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235543680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176848.076614:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176848.076618:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176848.076621:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235543680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176848.076640:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176848.076643:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176848.076645:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176848.076647:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176848.076648:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176848.076650:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176848.076652:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11522 00000400:00000200:0.0:1786176848.076657:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176848.076660:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176848.076662:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176848.076664:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076666:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176848.076668:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076673:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176848.076677:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176848.076680:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176848.076683:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176848.076684:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176848.076687:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176848.076689:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176848.076695:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076699:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176848.076703:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076720:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176848.076721:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176848.076723:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076726:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176848.076730:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176848.076736:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176848.076740:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.076745:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176848.076752:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176848.076758:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.076763:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176848.076771:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176848.076778:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176848.076783:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176848.076788:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.076793:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176848.076796:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176848.076799:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176848.076806:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786176848.076812:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176848.076825:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235543808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176848.076855:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176848.076859:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176848.076868:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235543808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176848.076878:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176848.076882:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176848.076888:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176848.076890:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176848.076892:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176848.076894:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176848.076900:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11522 00000400:00000200:0.0:1786176848.076905:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176848.076907:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176848.076909:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176848.076915:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076919:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176848.076923:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076928:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176848.076931:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176848.076935:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176848.076936:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176848.076940:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176848.076945:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176848.076950:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176848.076957:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076961:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176848.076967:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076972:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176848.076975:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176848.076981:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176848.076988:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176848.076994:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176848.076999:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176848.077004:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.077009:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176848.077013:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176848.077017:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.077032:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176848.077033:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000010:0.0:1786176848.077039:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000800:00000200:1.1:1786176848.077040:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000020:00000040:0.0:1786176848.077046:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176848.077053:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176848.077057:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.077060:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176848.077064:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176848.077067:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176848.077074:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176848.077083:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176848.077094:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235543936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176848.077114:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176848.077118:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176848.077129:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235543936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176848.077144:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176848.077151:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176848.077159:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176848.077165:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176848.077171:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176848.077179:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176848.077185:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11522 00000400:00000200:0.0:1786176848.077194:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176848.077201:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176848.077207:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176848.077210:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.077214:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176848.077217:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.077224:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176848.077230:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176848.077232:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176848.077237:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176848.077242:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176848.077248:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176848.077255:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176848.077265:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.077273:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176848.077279:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.077287:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176848.077295:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176848.077304:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176848.077312:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176848.077317:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176848.077322:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176848.077327:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.077329:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176848.077334:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176848.077339:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.077341:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176848.077345:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176848.077350:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176848.077355:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176848.077386:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.077389:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176848.077391:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176848.077395:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176848.077402:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176848.077411:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176848.077423:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235544064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176848.077443:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176848.077449:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176848.077461:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235544064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176848.077500:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176848.077507:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176848.077513:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176848.077519:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176848.077523:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176848.077532:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176848.077541:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11522 00000400:00000200:0.0:1786176848.077553:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176848.077562:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176848.077568:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176848.077577:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.077583:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176848.077589:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.077595:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176848.077599:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176848.077604:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176848.077610:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176848.077615:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176848.077622:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176848.077628:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176848.077638:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.077643:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176848.077650:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176848.077655:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176848.077661:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176848.077667:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176848.077672:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176848.077678:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176848.077684:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176848.077688:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.077692:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176848.077696:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176848.077700:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.077701:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176848.077723:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176848.077731:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176848.077735:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176848.077738:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176848.077741:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176848.077744:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176848.077749:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176848.077758:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176848.077765:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176848.077775:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235544192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176848.077804:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176848.077807:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176848.077814:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235544192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176848.077828:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176848.077835:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11532) 00000800:00000200:1.1:1786176848.077885:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176848.077889:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176848.078089:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176848.078091:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176848.078304:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176848.078306:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176848.078644:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176848.078646:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1786176848.588464:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176848.588472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176848.588475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176848.588481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176848.588487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176848.588491:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176848.588495:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176848.588498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176848.588502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176848.972468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176848.972474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176848.972484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176848.972491:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176848.972495:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176848.972499:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176848.972506:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176848.972511:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176848.972515:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176848.972518:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176848.972522:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176848.972523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176848.972588:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176848.972591:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176848.972594:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176849.612482:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176849.612490:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176849.612493:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176849.612499:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176849.612504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176849.612507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176849.612510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176849.612514:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176849.612518:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176849.996442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176849.996447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176849.996453:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176849.996456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176849.996462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176849.996466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176849.996528:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176849.996533:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176849.996536:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176850.636493:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176850.636501:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176850.636504:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176850.636511:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176850.636517:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176850.636521:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176850.636524:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176850.636527:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176850.636531:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176851.020467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176851.020472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176851.020479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176851.020482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176851.020487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176851.020489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176851.020549:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176851.020552:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176851.020556:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786176851.596433:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786176851.596439:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786176851.596445:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1786176851.596599:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176851.596617:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786176851.596623:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786176851.596627:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176851.660475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176851.660484:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176851.660487:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176851.660494:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176851.660498:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176851.660502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176851.660505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176851.660508:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176851.660511:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176851.980476:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306198784, at 4306198848 00000100:00000010:0.0:1786176851.980494:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62e40. 00010000:00000001:0.0:1786176852.044456:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176852.044464:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176852.044467:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176852.044471:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176852.044478:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176852.044480:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176852.044483:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176852.044486:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176852.044489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176852.044490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176852.044493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176852.044496:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176852.044500:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176852.044501:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176852.044503:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176852.044505:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176852.044509:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176852.044512:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176852.044566:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176852.044569:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176852.044571:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176852.684445:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176852.684454:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176852.684457:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176852.684463:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176852.684467:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176852.684472:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176852.684475:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176852.684478:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176852.684481:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176853.068493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176853.068499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176853.068517:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176853.068521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176853.068527:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176853.068532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176853.068613:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176853.068619:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176853.068623:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786176853.196525:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176853.196536:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786176853.196539:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176853.196543:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176853.196547:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176853.196555:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11527 00000400:00000200:2.0:1786176853.196564:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176853.196571:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176853.196574:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176853.196578:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.196581:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176853.196584:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.196587:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176853.196591:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176853.196593:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786176853.196596:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176853.196600:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786176853.196602:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786176853.196605:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786176853.196608:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.196610:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176853.196612:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.196614:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176853.196617:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786176853.196621:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176853.196624:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176853.196627:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176853.196630:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176853.196634:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.196636:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176853.196639:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176853.196648:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736d80. 00000020:00000040:2.0:1786176853.196652:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786176853.196692:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176853.196695:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.196699:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176853.196702:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176853.196706:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176853.196715:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786176853.196722:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176853.196732:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736d80 x1872930235544320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176853.196743:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176853.196748:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176853.196754:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e4f7686] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736d80 x1872930235544320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176853.196778:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176853.196782:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176853.196785:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786176853.196787:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176853.196788:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176853.196792:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176853.196794:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11527 00000400:00000200:2.0:1786176853.196798:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176853.196801:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176853.196803:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176853.196807:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.196809:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176853.196812:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.196814:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176853.196817:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176853.196819:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786176853.196821:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176853.196823:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786176853.196825:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786176853.196827:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786176853.196831:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.196834:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176853.196838:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.196843:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176853.196848:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786176853.196863:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176853.196873:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176853.196878:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176853.196886:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176853.196892:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.196898:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176853.196903:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176853.196911:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.196918:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176853.196932:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737480. 00000020:00000040:2.0:1786176853.196937:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176853.196942:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176853.196945:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.196951:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176853.196955:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176853.196959:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176853.196969:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:2.0:1786176853.196977:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176853.196988:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737480 x1872930235544448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176853.197003:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176853.197008:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176853.197018:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046bba14a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737480 x1872930235544448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176853.197041:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176853.197048:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176853.197055:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786176853.197069:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176853.197073:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176853.197079:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176853.197087:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11527 00000400:00000200:2.0:1786176853.197094:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:1.1:1786176853.197100:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:2.0:1786176853.197104:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000200:1.1:1786176853.197107:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00080000:2.0:1786176853.197109:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176853.197116:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197130:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176853.197136:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197143:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176853.197147:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176853.197148:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786176853.197153:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176853.197158:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786176853.197162:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786176853.197168:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786176853.197176:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197178:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176853.197180:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197185:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176853.197189:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786176853.197197:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197204:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176853.197209:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176853.197214:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176853.197219:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.197221:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176853.197222:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176853.197228:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.197233:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176853.197240:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734700. 00000020:00000040:2.0:1786176853.197247:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176853.197252:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176853.197255:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.197259:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176853.197264:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176853.197268:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176853.197276:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000001:2.0:1786176853.197282:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176853.197294:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734700 x1872930235544576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176853.197315:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176853.197320:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176853.197331:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088313762] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734700 x1872930235544576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176853.197350:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176853.197356:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176853.197426:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786176853.197430:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176853.197434:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176853.197441:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176853.197448:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11527 00000400:00000200:2.0:1786176853.197459:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176853.197469:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176853.197475:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176853.197481:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197486:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176853.197491:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197495:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176853.197500:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176853.197502:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786176853.197506:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176853.197510:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786176853.197516:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786176853.197522:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786176853.197530:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197534:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176853.197537:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197543:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176853.197550:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786176853.197558:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197566:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176853.197570:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176853.197575:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176853.197581:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.197586:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176853.197591:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176853.197597:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.197603:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176853.197610:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000020:00000040:2.0:1786176853.197615:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176853.197621:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176853.197625:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.197630:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176853.197635:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176853.197640:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176853.197648:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000001:2.0:1786176853.197673:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176853.197686:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235544704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176853.197706:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176853.197712:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176853.197723:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235544704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176853.197739:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176853.197745:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176853.197753:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786176853.197758:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176853.197762:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176853.197766:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176853.197773:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11527 00000400:00000200:2.0:1786176853.197783:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:1.1:1786176853.197783:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176853.197791:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000400:00000200:2.0:1786176853.197794:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176853.197800:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176853.197808:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197812:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176853.197819:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197823:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176853.197828:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176853.197830:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786176853.197834:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176853.197839:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786176853.197842:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786176853.197847:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786176853.197856:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197860:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176853.197867:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197872:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176853.197878:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786176853.197884:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176853.197890:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176853.197896:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176853.197901:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176853.197908:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.197913:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176853.197917:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176853.197925:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.197931:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176853.197939:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786176853.197946:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176853.197951:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176853.197954:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176853.197960:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176853.197962:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176853.197966:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176853.197974:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786176853.197982:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176853.197991:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235544832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176853.198022:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176853.198028:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176853.198040:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235544832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176853.198056:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176853.198064:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11537) 00000800:00000200:1.1:1786176853.198197:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176853.198199:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176853.198549:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176853.198551:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176853.198799:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176853.198802:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1786176853.708444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176853.708453:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176853.708456:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176853.708462:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176853.708467:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176853.708471:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176853.708483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176853.708486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176853.708489:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176854.092543:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176854.092551:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176854.092559:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176854.092561:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176854.092566:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176854.092568:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176854.092714:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176854.092720:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176854.092726:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176854.732487:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176854.732497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176854.732500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176854.732508:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176854.732513:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176854.732520:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176854.732525:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176854.732529:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176854.732534:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176855.116493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176855.116500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176855.116508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176855.116512:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176855.116519:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176855.116523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176855.116615:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176855.116620:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176855.116625:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176855.756476:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176855.756489:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176855.756494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176855.756503:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176855.756509:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176855.756513:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176855.756515:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176855.756518:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176855.756522:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176856.140523:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176856.140532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176856.140542:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176856.140547:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176856.140555:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176856.140560:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176856.140646:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176856.140651:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176856.140656:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176856.780539:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176856.780551:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176856.780555:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176856.780563:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176856.780567:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176856.780572:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176856.780576:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176856.780579:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176856.780584:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176857.100478:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306203904, at 4306203968 00000100:00000010:0.0:1786176857.100495:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62960. 00010000:00000001:0.0:1786176857.164474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176857.164478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176857.164484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176857.164487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176857.164490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176857.164493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176857.164553:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176857.164558:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176857.164562:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176857.804469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176857.804478:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176857.804481:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176857.804487:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176857.804491:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176857.804495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176857.804498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176857.804502:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176857.804506:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176858.188529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176858.188539:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176858.188550:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176858.188554:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176858.188559:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176858.188561:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176858.188650:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176858.188654:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176858.188657:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176858.316526:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176858.316541:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176858.316544:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176858.316549:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176858.316555:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176858.316564:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11532 00000400:00000200:0.0:1786176858.316576:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176858.316583:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176858.316587:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176858.316591:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.316595:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176858.316599:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.316602:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176858.316641:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176858.316644:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176858.316648:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176858.316652:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176858.316655:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176858.316658:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176858.316662:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.316664:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176858.316666:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.316668:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176858.316671:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176858.316675:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176858.316678:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176858.316682:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176858.316686:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176858.316690:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.316692:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176858.316695:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176858.316707:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176858.316711:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176858.316716:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176858.316718:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.316722:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176858.316726:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176858.316732:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176858.316744:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176858.316752:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176858.316764:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235544960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176858.316785:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176858.316795:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176858.316801:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235544960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176858.316832:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176858.316835:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176858.316839:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176858.316841:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176858.316843:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176858.316846:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176858.316849:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11532 00000400:00000200:0.0:1786176858.316852:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176858.316856:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176858.316859:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176858.316863:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.316865:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176858.316868:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.316871:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176858.316874:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176858.316876:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176858.316877:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176858.316881:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176858.316884:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176858.316889:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176858.316896:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.316902:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176858.316907:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.316914:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176858.316921:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176858.316928:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176858.316937:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176858.316941:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176858.316952:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176858.316957:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.316961:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176858.316969:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176858.316976:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.316983:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176858.316989:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176858.316994:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176858.317000:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176858.317002:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.317005:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176858.317008:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176858.317009:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176858.317015:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176858.317023:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176858.317040:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235545088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176858.317057:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176858.317063:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176858.317070:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235545088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176858.317086:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176858.317092:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176858.317100:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176858.317105:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176858.317109:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176858.317115:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176858.317124:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11532 00000400:00000200:0.0:1786176858.317146:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176858.317156:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176858.317160:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176858.317168:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317172:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176858.317178:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:1.1:1786176858.317182:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786176858.317184:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:1.1:1786176858.317189:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000020:00000001:0.0:1786176858.317190:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176858.317193:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176858.317197:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176858.317201:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176858.317207:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176858.317211:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176858.317216:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317221:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176858.317228:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317231:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176858.317237:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176858.317242:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317248:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176858.317252:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176858.317257:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176858.317261:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.317267:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176858.317271:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176858.317278:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.317282:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176858.317290:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176858.317296:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176858.317303:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176858.317307:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.317310:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176858.317315:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176858.317318:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176858.317324:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176858.317329:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176858.317336:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235545216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176858.317351:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176858.317389:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176858.317395:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235545216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176858.317409:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176858.317415:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176858.317426:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176858.317430:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176858.317435:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176858.317443:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176858.317453:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11532 00000400:00000200:0.0:1786176858.317462:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176858.317470:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176858.317476:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176858.317485:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317492:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176858.317499:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317507:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176858.317512:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176858.317517:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176858.317522:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176858.317528:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176858.317534:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176858.317542:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176858.317553:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317560:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176858.317568:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317573:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176858.317580:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176858.317589:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317596:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176858.317601:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176858.317619:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176858.317625:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.317629:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176858.317635:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176858.317641:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.317646:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176858.317654:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176858.317659:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176858.317666:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176858.317671:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.317678:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176858.317682:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176858.317686:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176858.317693:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176858.317699:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176858.317709:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235545344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176858.317724:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176858.317727:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176858.317731:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235545344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176858.317751:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176858.317756:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176858.317767:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176858.317771:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176858.317774:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176858.317781:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176858.317788:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11532 00000400:00000200:0.0:1786176858.317800:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176858.317808:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176858.317813:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176858.317821:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317826:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176858.317830:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317835:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176858.317840:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176858.317843:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176858.317847:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176858.317849:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176858.317854:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176858.317859:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176858.317867:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317872:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176858.317879:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317884:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176858.317891:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176858.317899:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176858.317906:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176858.317911:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176858.317916:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176858.317923:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.317928:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176858.317934:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176858.317941:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.317946:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176858.317955:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176858.317961:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176858.317967:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176858.317972:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176858.317976:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176858.317979:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176858.317984:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176858.317985:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176858.317990:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 02000000:00000010:0.0:1786176858.317992:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176858.318001:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176858.318011:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235545472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176858.318031:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176858.318035:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176858.318045:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235545472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176858.318069:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176858.318075:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11542) 00000800:00000200:1.1:1786176858.318285:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176858.318287:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176858.318575:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176858.318577:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176858.318816:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176858.318818:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1786176858.828473:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176858.828482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176858.828485:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176858.828491:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176858.828495:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176858.828498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176858.828501:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176858.828504:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176858.828509:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176859.212516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176859.212524:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176859.212532:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176859.212541:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176859.212548:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176859.212556:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176859.212562:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176859.212565:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176859.212575:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176859.212595:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176859.212620:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176859.212623:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176859.212691:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176859.212696:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176859.212700:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176859.852531:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176859.852539:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176859.852541:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176859.852546:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176859.852550:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176859.852554:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176859.852557:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176859.852560:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176859.852564:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176860.236537:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176860.236546:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176860.236558:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176860.236563:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176860.236575:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176860.236584:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176860.236670:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176860.236674:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176860.236677:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176860.876461:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176860.876468:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176860.876470:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176860.876475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176860.876479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176860.876482:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176860.876485:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176860.876487:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176860.876492:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176861.260448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176861.260452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176861.260457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176861.260461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176861.260466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176861.260470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176861.260558:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176861.260564:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176861.260568:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176861.900456:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176861.900465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176861.900468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176861.900474:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176861.900480:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176861.900484:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176861.900488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176861.900493:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176861.900497:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176862.220463:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306209024, at 4306209088 00000100:00000010:0.0:1786176862.220480:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62000. 00010000:00000001:0.0:1786176862.284464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176862.284472:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176862.284477:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176862.284484:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176862.284491:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176862.284496:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176862.284500:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176862.284502:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176862.284507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176862.284508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176862.284511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176862.284512:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176862.284514:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176862.284516:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176862.284518:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176862.284519:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176862.284520:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176862.284522:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176862.284589:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176862.284592:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176862.284594:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176862.924460:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176862.924468:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176862.924471:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176862.924477:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176862.924483:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176862.924486:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176862.924489:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176862.924492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176862.924495:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176863.308467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176863.308471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176863.308476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176863.308482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176863.308489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176863.308491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176863.308540:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176863.308544:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176863.308546:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176863.436491:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176863.436500:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176863.436502:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176863.436506:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176863.436509:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176863.436515:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11537 00000400:00000200:3.0:1786176863.436521:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176863.436526:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176863.436529:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176863.436531:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436534:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176863.436536:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436537:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176863.436541:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176863.436542:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176863.436545:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176863.436547:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176863.436549:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176863.436550:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176863.436552:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436554:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176863.436555:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436571:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176863.436574:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176863.436577:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436578:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176863.436580:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176863.436583:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176863.436585:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.436586:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176863.436589:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176863.436597:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176863.436600:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176863.436603:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176863.436604:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.436607:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176863.436609:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176863.436613:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176863.436619:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176863.436624:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176863.436632:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235545600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176863.436644:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176863.436649:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176863.436655:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235545600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176863.436680:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176863.436685:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176863.436689:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176863.436690:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176863.436692:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176863.436695:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176863.436697:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11537 00000400:00000200:3.0:1786176863.436701:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176863.436703:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176863.436706:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176863.436708:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436710:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176863.436712:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436714:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176863.436715:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176863.436716:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176863.436718:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176863.436721:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176863.436724:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176863.436728:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176863.436733:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436735:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176863.436739:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436743:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176863.436747:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176863.436753:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436761:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176863.436765:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176863.436772:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176863.436778:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.436781:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176863.436787:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176863.436791:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.436793:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176863.436795:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176863.436798:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176863.436802:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176863.436804:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.436809:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176863.436811:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176863.436815:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176863.436821:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176863.436826:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176863.436836:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235545728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176863.436846:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176863.436850:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176863.436853:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235545728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176863.436871:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176863.436875:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176863.436884:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176863.436888:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176863.436892:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176863.436899:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176863.436906:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11537 00000400:00000200:3.0:1786176863.436913:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176863.436922:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176863.436928:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176863.436930:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436932:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176863.436936:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436939:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176863.436942:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176863.436945:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176863.436949:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176863.436951:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176863.436952:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176863.436955:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176863.436961:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436963:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176863.436968:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436971:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176863.436973:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176863.436978:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176863.436982:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176863.436986:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176863.436992:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176863.436996:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.437002:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176863.437005:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176863.437012:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.437016:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176863.437024:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176863.437031:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176863.437036:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176863.437040:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.437042:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176863.437046:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176863.437048:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176863.437055:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176863.437060:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176863.437060:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00100000:3.0:1786176863.437066:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235545856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1786176863.437067:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1786176863.437079:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176863.437083:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176863.437088:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235545856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176863.437097:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176863.437100:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176863.437103:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176863.437106:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176863.437109:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176863.437115:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176863.437119:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11537 00000400:00000200:3.0:1786176863.437127:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176863.437132:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176863.437136:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176863.437141:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.437146:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176863.437150:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.437154:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176863.437157:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176863.437160:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176863.437164:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176863.437166:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176863.437171:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176863.437178:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176863.437187:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.437194:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176863.437200:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.437205:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176863.437211:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176863.437217:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176863.437222:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176863.437225:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176863.437230:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176863.437236:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.437240:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176863.437244:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176863.437249:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.437253:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176863.437258:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176863.437263:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176863.437266:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176863.437269:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.437272:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176863.437277:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176863.437280:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176863.437285:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176863.437291:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176863.437300:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235545984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176863.437308:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176863.437310:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176863.437318:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235545984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176863.437337:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176863.437342:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176863.437350:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176863.437353:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176863.437354:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176863.437390:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176863.437394:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11537 00000400:00000200:3.0:1786176863.437403:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176863.437412:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176863.437419:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176863.437426:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.437431:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176863.437436:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.437443:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176863.437447:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176863.437449:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176863.437452:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176863.437457:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176863.437461:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176863.437466:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176863.437473:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.437476:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176863.437479:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176863.437484:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176863.437487:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176863.437494:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176863.437498:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176863.437501:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176863.437507:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176863.437511:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.437514:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176863.437517:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176863.437524:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.437527:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176863.437533:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176863.437538:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176863.437542:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176863.437545:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176863.437548:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176863.437551:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176863.437553:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176863.437573:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176863.437578:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176863.437588:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235546112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176863.437596:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176863.437599:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176863.437605:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235546112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176863.437618:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176863.437623:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11548) 00000800:00000200:1.1:1786176863.437738:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176863.437740:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176863.437928:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176863.437929:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176863.438164:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176863.438166:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176863.438390:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176863.438392:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1786176863.948444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176863.948452:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176863.948455:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176863.948461:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176863.948465:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176863.948468:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176863.948471:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176863.948473:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176863.948476:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176864.332455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176864.332460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176864.332467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176864.332470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176864.332474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176864.332476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176864.332535:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176864.332538:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176864.332541:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176864.972454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176864.972463:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176864.972464:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176864.972469:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176864.972473:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176864.972486:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176864.972488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176864.972492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176864.972495:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176865.356451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176865.356456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176865.356461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176865.356464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176865.356467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176865.356470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176865.356528:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176865.356532:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176865.356535:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176865.996449:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176865.996456:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176865.996458:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176865.996463:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176865.996467:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176865.996470:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176865.996472:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176865.996475:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176865.996477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176866.380475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176866.380479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176866.380486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176866.380489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176866.380494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176866.380495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176866.380556:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176866.380560:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176866.380563:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176867.020473:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176867.020479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176867.020481:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176867.020486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176867.020489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176867.020492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176867.020494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176867.020497:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176867.020499:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176867.340477:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306214144, at 4306214208 00000100:00000010:0.0:1786176867.340495:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62600. 00010000:00000001:0.0:1786176867.404452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176867.404460:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176867.404468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176867.404473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176867.404481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176867.404484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176867.404573:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176867.404578:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176867.404582:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176868.044470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176868.044491:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176868.044493:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176868.044500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176868.044503:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176868.044507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176868.044509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176868.044524:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176868.044528:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176868.428459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176868.428466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176868.428474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176868.428479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176868.428487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176868.428491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176868.428559:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176868.428563:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176868.428566:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176868.556420:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176868.556429:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176868.556431:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176868.556434:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176868.556438:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176868.556444:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11543 00000400:00000200:3.0:1786176868.556452:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176868.556457:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176868.556459:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176868.556462:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556465:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176868.556467:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556468:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176868.556471:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176868.556473:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176868.556475:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176868.556477:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176868.556479:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176868.556480:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176868.556483:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556484:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176868.556485:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556487:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176868.556489:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176868.556491:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556493:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176868.556495:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176868.556497:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176868.556499:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.556501:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176868.556503:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176868.556535:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786176868.556539:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176868.556542:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176868.556543:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.556546:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176868.556549:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176868.556557:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176868.556563:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176868.556568:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176868.556576:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235546240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176868.556584:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176868.556588:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176868.556591:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235546240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176868.556610:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176868.556612:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176868.556615:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176868.556616:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176868.556617:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176868.556619:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176868.556621:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11543 00000400:00000200:3.0:1786176868.556623:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176868.556625:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176868.556627:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176868.556629:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556631:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176868.556633:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556634:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176868.556636:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176868.556638:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176868.556639:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176868.556641:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176868.556643:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176868.556646:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176868.556649:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556652:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176868.556654:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556657:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176868.556659:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176868.556663:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556669:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176868.556674:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176868.556688:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176868.556694:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.556698:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176868.556703:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176868.556709:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.556712:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176868.556717:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786176868.556723:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176868.556728:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176868.556731:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.556734:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176868.556735:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176868.556736:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176868.556740:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176868.556744:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176868.556752:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235546368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176868.556763:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176868.556767:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176868.556773:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235546368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176868.556783:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176868.556786:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176868.556792:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176868.556794:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176868.556798:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176868.556800:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176868.556805:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11543 00000400:00000200:3.0:1786176868.556813:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176868.556819:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176868.556824:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176868.556830:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556833:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176868.556837:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556838:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176868.556843:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176868.556845:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176868.556847:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176868.556849:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176868.556850:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176868.556853:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176868.556860:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556862:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176868.556867:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556869:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176868.556872:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176868.556877:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176868.556881:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176868.556885:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176868.556889:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176868.556892:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.556895:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176868.556898:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176868.556903:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.556906:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176868.556911:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786176868.556915:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176868.556920:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176868.556922:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.556924:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176868.556926:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176868.556941:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176868.556941:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176868.556947:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 02000000:00000010:3.0:1786176868.556948:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176868.556954:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176868.556961:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235546496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176868.556976:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176868.556980:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176868.556986:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235546496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176868.556999:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176868.557003:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176868.557010:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176868.557013:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176868.557015:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176868.557020:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176868.557025:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11543 00000400:00000200:3.0:1786176868.557033:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176868.557040:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176868.557043:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176868.557049:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.557051:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176868.557056:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.557060:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176868.557064:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176868.557067:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176868.557069:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176868.557072:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176868.557076:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176868.557078:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176868.557080:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.557082:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176868.557085:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.557086:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176868.557088:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176868.557095:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176868.557100:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176868.557107:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176868.557110:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176868.557115:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.557120:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176868.557123:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176868.557129:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.557131:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176868.557137:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176868.557141:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176868.557145:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176868.557147:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.557151:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176868.557154:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176868.557156:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176868.557161:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176868.557166:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176868.557175:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235546624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176868.557186:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176868.557190:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176868.557197:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235546624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176868.557223:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176868.557230:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176868.557240:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176868.557245:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176868.557249:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176868.557255:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176868.557260:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11543 00000400:00000200:3.0:1786176868.557268:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176868.557272:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176868.557276:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176868.557281:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.557286:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176868.557290:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.557293:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176868.557296:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176868.557298:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176868.557301:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176868.557304:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176868.557306:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176868.557308:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176868.557314:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.557319:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176868.557325:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176868.557329:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176868.557334:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176868.557338:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176868.557343:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176868.557346:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176868.557350:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176868.557355:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.557376:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176868.557377:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176868.557379:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.557381:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176868.557384:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176868.557386:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176868.557388:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176868.557389:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176868.557390:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176868.557391:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176868.557394:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176868.557399:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176868.557404:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176868.557411:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235546752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176868.557426:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176868.557427:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176868.557430:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235546752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176868.557444:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176868.557450:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11553) 00000800:00000200:1.1:1786176868.557559:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176868.557560:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176868.557784:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176868.557785:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176868.557964:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176868.557966:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786176868.558204:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176868.558206:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1786176869.068480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176869.068490:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176869.068493:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176869.068500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176869.068520:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176869.068524:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176869.068528:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176869.068532:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176869.068536:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176869.452479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176869.452482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176869.452485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176869.452488:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176869.452489:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176869.452492:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176869.452494:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176869.452497:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176869.452528:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176869.452530:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176869.452533:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176869.452534:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176869.452581:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176869.452584:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176869.452586:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176870.092464:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176870.092471:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176870.092474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176870.092479:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176870.092483:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176870.092486:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176870.092488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176870.092490:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176870.092508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176870.476461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176870.476467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176870.476473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176870.476478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176870.476485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176870.476503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176870.476569:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176870.476573:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176870.476578:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176871.116467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176871.116474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176871.116476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176871.116497:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176871.116502:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176871.116506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176871.116509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176871.116511:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176871.116513:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176871.500449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176871.500456:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176871.500459:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176871.500462:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176871.500465:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176871.500468:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176871.500469:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176871.500471:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176871.500475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176871.500476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176871.500499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176871.500501:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176871.500502:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176871.500503:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176871.500504:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176871.500505:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176871.500506:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176871.500508:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176871.500555:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176871.500559:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176871.500561:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176872.140518:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176872.140525:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176872.140526:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176872.140532:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176872.140537:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176872.140542:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176872.140545:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176872.140547:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176872.140550:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176872.460501:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306219264, at 4306219328 00000100:00000010:0.0:1786176872.460520:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ea0. 00010000:00000001:0.0:1786176872.524532:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176872.524539:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176872.524545:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176872.524549:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176872.524554:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176872.524557:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176872.524630:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176872.524634:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176872.524638:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176873.164509:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176873.164518:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176873.164521:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176873.164528:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176873.164533:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176873.164536:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176873.164539:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176873.164542:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176873.164545:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176873.548501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176873.548506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176873.548512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176873.548524:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176873.548531:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176873.548538:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176873.548618:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176873.548624:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176873.548628:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176873.676525:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176873.676541:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176873.676544:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176873.676549:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176873.676555:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176873.676564:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11548 00000400:00000200:3.0:1786176873.676575:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176873.676583:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176873.676587:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176873.676591:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676595:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176873.676599:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676602:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176873.676607:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176873.676609:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176873.676612:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176873.676616:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176873.676619:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176873.676623:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176873.676626:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676630:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176873.676633:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676635:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176873.676638:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176873.676643:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676645:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176873.676648:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176873.676651:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176873.676654:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.676656:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176873.676658:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176873.676670:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176873.676673:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176873.676677:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176873.676678:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.676682:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176873.676684:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176873.676689:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176873.676697:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176873.676704:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176873.676714:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235546880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176873.676724:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176873.676728:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176873.676732:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235546880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176873.676755:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176873.676757:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176873.676760:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176873.676761:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176873.676763:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176873.676765:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176873.676767:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11548 00000400:00000200:3.0:1786176873.676770:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176873.676772:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176873.676774:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176873.676776:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676778:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176873.676779:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676781:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176873.676783:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176873.676785:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176873.676786:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176873.676788:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176873.676789:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176873.676791:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176873.676793:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676794:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176873.676796:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676798:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176873.676801:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176873.676804:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676809:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176873.676812:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176873.676819:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176873.676824:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.676827:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176873.676833:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176873.676838:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.676842:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176873.676847:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176873.676851:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176873.676854:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176873.676858:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.676863:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176873.676865:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176873.676868:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176873.676871:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176873.676876:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176873.676881:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235547008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176873.676888:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176873.676891:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176873.676895:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235547008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176873.676904:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176873.676908:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176873.676911:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176873.676914:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176873.676917:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176873.676920:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176873.676925:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11548 00000400:00000200:3.0:1786176873.676928:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176873.676930:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176873.676931:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176873.676934:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676944:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176873.676949:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676952:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176873.676956:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176873.676958:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176873.676960:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176873.676962:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176873.676966:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176873.676971:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176873.676978:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676981:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176873.676987:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.676989:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176873.676994:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176873.677000:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176873.677002:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176873.677003:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176873.677007:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176873.677011:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.677013:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176873.677014:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176873.677016:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.677017:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176873.677023:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176873.677026:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176873.677028:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176873.677030:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.677034:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176873.677035:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176873.677037:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176873.677040:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176873.677044:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176873.677050:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235547136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176873.677064:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176873.677068:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176873.677075:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235547136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176873.677084:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176873.677088:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176873.677093:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176873.677095:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176873.677097:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176873.677102:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176873.677106:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11548 00000400:00000200:3.0:1786176873.677111:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176873.677113:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176873.677115:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176873.677118:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.677119:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176873.677125:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.677129:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176873.677135:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:1.1:1786176873.677135:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786176873.677138:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000800:00000200:1.1:1786176873.677141:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000020:00000001:3.0:1786176873.677142:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176873.677147:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176873.677153:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176873.677158:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176873.677165:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.677169:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176873.677176:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.677181:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176873.677186:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176873.677192:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176873.677198:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176873.677202:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176873.677207:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176873.677210:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.677213:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176873.677218:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176873.677224:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.677227:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176873.677233:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176873.677239:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176873.677245:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176873.677248:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.677251:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176873.677253:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176873.677255:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176873.677262:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176873.677268:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176873.677276:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235547264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176873.677284:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176873.677287:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176873.677290:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235547264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176873.677301:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176873.677306:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176873.677311:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176873.677314:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176873.677317:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176873.677321:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176873.677325:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11548 00000400:00000200:3.0:1786176873.677333:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176873.677339:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176873.677341:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176873.677345:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.677348:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176873.677352:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.677378:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176873.677381:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176873.677381:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176873.677382:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176873.677384:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176873.677386:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176873.677388:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176873.677391:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.677394:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176873.677399:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176873.677402:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176873.677407:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176873.677413:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176873.677416:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176873.677419:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176873.677423:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176873.677427:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.677430:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176873.677433:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176873.677437:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.677440:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176873.677444:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176873.677449:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176873.677453:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176873.677456:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176873.677470:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176873.677472:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176873.677474:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176873.677481:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176873.677486:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176873.677494:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235547392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176873.677515:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176873.677518:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176873.677525:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235547392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176873.677535:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176873.677539:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11558) 00000800:00000200:1.1:1786176873.677722:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176873.677724:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176873.677985:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176873.677988:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176873.678206:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176873.678208:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786176873.678417:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176873.678419:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1786176874.188529:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176874.188537:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176874.188540:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176874.188547:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176874.188553:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176874.188559:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176874.188562:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176874.188566:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176874.188570:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176874.572512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176874.572518:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176874.572525:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176874.572527:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176874.572533:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176874.572535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176874.572616:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176874.572621:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176874.572625:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176875.212491:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176875.212498:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176875.212500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176875.212504:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176875.212507:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176875.212510:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176875.212512:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176875.212514:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176875.212517:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176875.596519:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176875.596525:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176875.596538:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176875.596542:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176875.596548:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176875.596550:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176875.596630:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176875.596634:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176875.596636:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176876.236594:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176876.236600:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176876.236602:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176876.236607:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176876.236611:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176876.236615:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176876.236617:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176876.236622:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176876.236625:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176876.620505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176876.620511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176876.620518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176876.620523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176876.620530:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176876.620535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176876.620606:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176876.620611:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176876.620615:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176877.260469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176877.260478:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176877.260481:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176877.260486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176877.260490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176877.260495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176877.260497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176877.260500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176877.260503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176877.580508:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306224384, at 4306224448 00000100:00000010:0.0:1786176877.580531:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c624e0. 00010000:00000001:0.0:1786176877.644473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176877.644478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176877.644484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176877.644487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176877.644491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176877.644492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176877.644551:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176877.644558:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176877.644562:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176878.284494:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176878.284503:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176878.284505:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176878.284511:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176878.284515:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176878.284519:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176878.284521:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176878.284524:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176878.284526:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176878.668442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176878.668447:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176878.668457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176878.668461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176878.668467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176878.668470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176878.668516:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176878.668520:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176878.668523:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176878.796478:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176878.796487:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176878.796489:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176878.796492:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176878.796495:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176878.796502:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11553 00000400:00000200:0.0:1786176878.796509:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176878.796514:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176878.796516:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176878.796518:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796521:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176878.796523:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796525:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176878.796528:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176878.796529:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176878.796531:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176878.796534:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176878.796536:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176878.796538:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176878.796540:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796541:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176878.796542:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796544:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176878.796546:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176878.796549:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796550:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176878.796553:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176878.796555:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176878.796557:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.796559:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176878.796561:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176878.796569:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786176878.796571:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176878.796574:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176878.796576:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.796579:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176878.796581:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176878.796585:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176878.796592:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176878.796599:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176878.796608:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235547520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176878.796619:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176878.796624:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176878.796629:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235547520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176878.796647:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176878.796650:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176878.796652:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176878.796653:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176878.796654:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176878.796657:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176878.796658:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11553 00000400:00000200:0.0:1786176878.796662:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176878.796664:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176878.796666:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176878.796670:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796672:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176878.796673:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796675:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176878.796677:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176878.796681:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176878.796683:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176878.796687:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176878.796691:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176878.796693:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176878.796704:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796707:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176878.796711:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796716:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176878.796718:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176878.796723:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796726:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176878.796731:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176878.796738:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176878.796745:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.796750:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176878.796755:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176878.796758:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.796763:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176878.796771:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786176878.796775:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176878.796779:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176878.796781:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.796786:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176878.796787:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176878.796789:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176878.796793:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786176878.796798:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176878.796809:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235547648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176878.796819:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176878.796823:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176878.796831:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235547648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176878.796844:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176878.796848:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176878.796853:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176878.796855:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176878.796857:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176878.796862:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176878.796868:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11553 00000400:00000200:0.0:1786176878.796876:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176878.796882:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176878.796886:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176878.796894:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796899:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176878.796907:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796912:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176878.796917:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176878.796920:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176878.796924:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176878.796927:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176878.796931:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176878.796935:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176878.796943:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796945:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176878.796947:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796950:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176878.796955:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176878.796960:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176878.796964:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176878.796969:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176878.796973:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176878.796977:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.796981:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176878.796982:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176878.796986:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.796988:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176878.796993:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786176878.796999:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000800:00000200:1.1:1786176878.797003:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:0.0:1786176878.797004:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176878.797007:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176878.797009:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1786176878.797010:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176878.797013:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176878.797016:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176878.797021:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176878.797026:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176878.797037:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235547776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176878.797051:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176878.797054:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176878.797062:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235547776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176878.797094:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176878.797100:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176878.797110:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176878.797114:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176878.797118:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176878.797125:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176878.797130:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11553 00000400:00000200:0.0:1786176878.797141:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176878.797148:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176878.797152:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176878.797168:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.797173:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176878.797179:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.797184:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176878.797186:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176878.797188:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176878.797190:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176878.797194:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176878.797199:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176878.797206:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176878.797214:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.797220:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176878.797226:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.797232:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176878.797240:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176878.797247:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176878.797252:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176878.797256:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176878.797260:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176878.797265:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.797269:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176878.797272:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176878.797278:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.797281:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176878.797287:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786176878.797291:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176878.797295:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176878.797297:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.797301:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176878.797303:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176878.797306:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176878.797319:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176878.797327:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176878.797338:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235547904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176878.797393:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176878.797398:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176878.797407:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235547904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176878.797429:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176878.797433:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176878.797440:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176878.797443:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176878.797446:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176878.797449:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176878.797452:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11553 00000400:00000200:0.0:1786176878.797460:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176878.797466:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176878.797469:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176878.797474:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.797479:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176878.797482:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.797487:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176878.797491:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176878.797495:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176878.797498:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176878.797501:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176878.797505:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176878.797508:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176878.797513:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.797518:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176878.797524:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176878.797529:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176878.797533:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176878.797538:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176878.797542:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176878.797548:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176878.797552:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176878.797557:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.797561:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176878.797564:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176878.797567:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.797571:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176878.797575:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786176878.797583:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176878.797588:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176878.797592:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176878.797598:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176878.797603:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176878.797608:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176878.797618:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176878.797626:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176878.797637:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235548032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176878.797658:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176878.797662:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176878.797668:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235548032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176878.797685:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176878.797690:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11563) 00000800:00000200:1.1:1786176878.797703:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176878.797704:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176878.797924:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176878.797926:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176878.798132:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176878.798134:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786176878.798394:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176878.798396:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1786176879.308458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176879.308469:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176879.308472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176879.308479:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176879.308484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176879.308488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176879.308492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176879.308496:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176879.308499:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176879.692538:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176879.692544:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176879.692551:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176879.692556:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176879.692558:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176879.692561:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176879.692563:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176879.692565:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176879.692566:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176879.692568:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176879.692570:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176879.692571:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176879.692621:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176879.692627:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176879.692631:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176880.332483:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176880.332492:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176880.332494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176880.332500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176880.332503:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176880.332513:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176880.332515:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176880.332518:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176880.332523:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176880.716472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176880.716475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176880.716480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176880.716482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176880.716485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176880.716487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176880.716547:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176880.716552:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176880.716557:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176881.356481:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176881.356494:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176881.356496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176881.356503:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176881.356508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176881.356513:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176881.356516:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176881.356520:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176881.356523:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176881.740457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176881.740465:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176881.740470:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176881.740475:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176881.740483:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176881.740486:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176881.740488:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176881.740491:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176881.740496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176881.740498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176881.740501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176881.740503:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176881.740504:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176881.740506:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176881.740507:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176881.740509:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176881.740510:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176881.740512:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176881.740579:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176881.740585:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176881.740589:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176882.380474:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176882.380480:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176882.380482:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176882.380487:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176882.380490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176882.380494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176882.380496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176882.380498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176882.380500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176882.700486:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306229504, at 4306229568 00000100:00000010:0.0:1786176882.700502:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c620c0. 00010000:00000001:0.0:1786176882.764483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176882.764495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176882.764505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176882.764508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176882.764515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176882.764520:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176882.764592:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176882.764596:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176882.764599:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176883.404441:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176883.404448:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176883.404450:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176883.404455:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176883.404458:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176883.404461:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176883.404464:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176883.404466:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176883.404469:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176883.788508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176883.788515:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176883.788527:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176883.788529:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176883.788534:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176883.788535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176883.788601:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176883.788606:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176883.788609:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176883.916494:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176883.916509:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176883.916512:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176883.916517:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176883.916524:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176883.916534:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11558 00000400:00000200:3.0:1786176883.916546:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176883.916554:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176883.916558:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176883.916561:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916564:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176883.916566:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916568:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176883.916572:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176883.916574:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176883.916578:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176883.916581:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176883.916583:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176883.916585:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176883.916587:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916588:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176883.916590:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916591:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176883.916593:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176883.916596:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916598:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176883.916600:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176883.916604:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176883.916606:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.916609:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176883.916613:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176883.916623:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176883.916627:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176883.916631:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176883.916633:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.916636:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176883.916639:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176883.916644:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176883.916651:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176883.916658:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176883.916667:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235548160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176883.916678:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176883.916682:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176883.916686:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235548160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176883.916707:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176883.916710:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176883.916713:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176883.916714:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176883.916715:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176883.916718:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176883.916719:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11558 00000400:00000200:3.0:1786176883.916721:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176883.916723:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176883.916725:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176883.916728:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916730:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176883.916732:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916734:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176883.916737:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176883.916738:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176883.916739:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176883.916741:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176883.916743:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176883.916745:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176883.916749:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916752:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176883.916756:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916761:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176883.916763:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176883.916774:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916776:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176883.916784:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176883.916793:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176883.916798:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.916802:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176883.916810:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176883.916817:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.916822:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176883.916827:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176883.916834:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176883.916838:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176883.916842:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.916847:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176883.916848:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176883.916850:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176883.916857:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176883.916864:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176883.916873:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235548288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176883.916883:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176883.916888:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176883.916896:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235548288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176883.916921:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176883.916929:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176883.916937:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176883.916941:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176883.916945:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176883.916951:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176883.916957:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11558 00000400:00000200:3.0:1786176883.916967:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176883.916973:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176883.916976:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176883.916983:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916986:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176883.916992:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.916997:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176883.917001:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176883.917004:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176883.917008:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176883.917011:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176883.917013:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176883.917015:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176883.917022:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917025:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176883.917030:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917033:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176883.917037:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176883.917043:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917046:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176883.917049:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176883.917052:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176883.917056:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.917058:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176883.917061:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176883.917063:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.917068:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176883.917072:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000010:3.0:1786176883.917075:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000800:00000200:1.1:1786176883.917076:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000020:00000040:3.0:1786176883.917079:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176883.917084:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176883.917086:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.917090:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176883.917094:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176883.917096:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176883.917101:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176883.917108:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176883.917119:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235548416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176883.917143:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176883.917148:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176883.917156:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235548416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176883.917169:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176883.917173:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176883.917180:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176883.917183:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176883.917186:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176883.917189:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176883.917193:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11558 00000400:00000200:3.0:1786176883.917198:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176883.917203:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176883.917205:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176883.917209:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917211:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176883.917215:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917219:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176883.917223:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176883.917227:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176883.917230:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176883.917233:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176883.917236:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176883.917240:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176883.917245:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917249:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176883.917254:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917259:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176883.917264:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176883.917270:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917275:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176883.917277:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176883.917280:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176883.917285:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.917289:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176883.917291:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176883.917297:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.917300:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176883.917306:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176883.917312:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176883.917317:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176883.917321:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.917326:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176883.917329:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176883.917332:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176883.917337:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176883.917343:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176883.917352:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235548544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176883.917436:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176883.917440:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176883.917448:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235548544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176883.917463:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176883.917467:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176883.917473:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176883.917477:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176883.917478:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176883.917481:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176883.917486:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11558 00000400:00000200:3.0:1786176883.917494:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176883.917502:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176883.917505:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176883.917512:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917516:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176883.917521:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917528:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176883.917533:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176883.917536:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176883.917540:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176883.917544:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176883.917549:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176883.917554:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176883.917560:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917564:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176883.917569:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917574:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176883.917578:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176883.917583:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176883.917587:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176883.917591:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176883.917596:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176883.917600:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.917605:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176883.917607:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176883.917609:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.917610:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176883.917615:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176883.917618:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176883.917622:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176883.917626:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176883.917629:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176883.917631:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176883.917634:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176883.917640:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176883.917644:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176883.917652:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235548672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176883.917668:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176883.917672:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176883.917682:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235548672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176883.917696:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176883.917702:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11568) 00000800:00000200:1.1:1786176883.917772:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176883.917774:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176883.917966:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176883.917968:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176883.918151:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176883.918153:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176883.918444:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176883.918446:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1786176884.428491:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176884.428503:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176884.428506:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176884.428514:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176884.428519:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176884.428524:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176884.428528:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176884.428531:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176884.428535:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176884.812450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176884.812456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176884.812464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176884.812467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176884.812474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176884.812477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176884.812546:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176884.812550:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176884.812554:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176885.452502:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176885.452525:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176885.452529:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176885.452536:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176885.452540:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176885.452544:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176885.452546:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176885.452549:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176885.452552:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176885.836468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176885.836473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176885.836480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176885.836483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176885.836487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176885.836490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176885.836543:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176885.836547:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176885.836549:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176886.476715:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176886.476729:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176886.476733:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176886.476742:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176886.476749:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176886.476756:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176886.476760:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176886.476765:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176886.476770:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176886.860461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176886.860467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176886.860484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176886.860488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176886.860497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176886.860499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176886.860559:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176886.860562:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176886.860565:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176887.501690:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176887.501698:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176887.501700:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176887.501706:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176887.501710:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176887.501713:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176887.501716:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176887.501719:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176887.501722:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176887.820463:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306234624, at 4306234688 00000100:00000010:0.0:1786176887.820481:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c622a0. 00010000:00000001:0.0:1786176887.884480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176887.884487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176887.884494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176887.884498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176887.884505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176887.884510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176887.884576:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176887.884580:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176887.884582:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176888.524729:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176888.524738:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176888.524741:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176888.524747:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176888.524752:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176888.524756:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176888.524759:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176888.524763:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176888.524767:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176888.908445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176888.908452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176888.908461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176888.908465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176888.908471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176888.908473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176888.908525:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176888.908531:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176888.908534:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176889.036443:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176889.036451:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176889.036454:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176889.036457:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176889.036460:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176889.036467:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11563 00000400:00000200:3.0:1786176889.036474:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176889.036486:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176889.036489:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176889.036491:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036494:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176889.036496:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036498:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176889.036501:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176889.036503:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176889.036505:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176889.036508:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176889.036510:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176889.036512:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176889.036514:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036515:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176889.036517:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036518:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176889.036520:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176889.036522:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036524:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176889.036526:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176889.036529:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176889.036531:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.036533:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176889.036535:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176889.036540:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786176889.036543:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176889.036547:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176889.036549:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.036552:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176889.036555:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176889.036560:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176889.036568:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176889.036575:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176889.036595:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235548800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176889.036607:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176889.036611:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176889.036615:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235548800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176889.036633:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176889.036635:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176889.036638:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176889.036639:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176889.036641:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176889.036643:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176889.036644:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11563 00000400:00000200:3.0:1786176889.036648:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176889.036650:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176889.036651:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176889.036654:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036655:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176889.036657:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036659:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176889.036661:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176889.036662:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176889.036663:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176889.036665:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176889.036666:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176889.036668:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176889.036671:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036673:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176889.036674:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036676:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176889.036678:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176889.036683:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036686:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176889.036689:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176889.036694:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176889.036699:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.036700:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176889.036706:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176889.036710:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.036715:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176889.036720:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786176889.036725:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176889.036741:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176889.036745:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.036749:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176889.036752:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176889.036757:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176889.036770:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176889.036789:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176889.036800:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235548928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176889.036849:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176889.036854:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176889.036864:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235548928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176889.036885:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176889.036889:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176889.036894:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176889.036899:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176889.036902:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176889.036910:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176889.036915:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11563 00000400:00000200:3.0:1786176889.036922:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176889.036928:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176889.036934:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176889.036943:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036949:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176889.036955:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036960:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176889.036966:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176889.036971:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176889.036974:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176889.036977:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000800:00000200:1.1:1786176889.036979:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:3.0:1786176889.036980:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176889.036986:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000200:1.1:1786176889.036987:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00000040:3.0:1786176889.036990:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.036995:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176889.036998:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037002:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176889.037004:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176889.037011:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037015:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176889.037018:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176889.037023:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176889.037029:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.037034:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176889.037037:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176889.037039:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.037042:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176889.037047:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786176889.037052:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176889.037056:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176889.037059:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.037063:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176889.037066:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176889.037069:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176889.037076:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176889.037082:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176889.037093:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235549056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176889.037123:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176889.037128:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176889.037135:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235549056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176889.037149:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176889.037156:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176889.037164:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176889.037167:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176889.037170:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176889.037174:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176889.037179:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11563 00000400:00000200:3.0:1786176889.037186:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176889.037193:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176889.037201:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176889.037208:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037213:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176889.037220:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037225:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176889.037239:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176889.037242:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176889.037247:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176889.037252:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176889.037257:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176889.037262:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176889.037273:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037276:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176889.037280:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037284:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176889.037288:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176889.037296:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037303:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176889.037325:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176889.037331:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176889.037337:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.037341:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176889.037347:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176889.037353:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.037387:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176889.037396:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176889.037402:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176889.037408:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176889.037411:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.037415:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176889.037418:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176889.037422:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176889.037432:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176889.037440:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176889.037448:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235549184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176889.037461:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176889.037462:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176889.037465:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235549184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176889.037488:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176889.037493:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176889.037499:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176889.037503:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176889.037507:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176889.037513:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176889.037519:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11563 00000400:00000200:3.0:1786176889.037528:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176889.037536:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176889.037541:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176889.037549:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037557:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176889.037562:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037567:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176889.037574:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176889.037578:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176889.037581:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176889.037585:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176889.037591:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176889.037597:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176889.037606:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037609:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176889.037614:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037618:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176889.037624:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176889.037630:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176889.037636:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176889.037639:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176889.037642:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176889.037648:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.037650:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176889.037654:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176889.037661:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.037666:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176889.037673:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176889.037677:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176889.037680:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176889.037683:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176889.037687:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176889.037690:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176889.037693:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176889.037698:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176889.037704:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176889.037712:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235549312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176889.037727:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176889.037731:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176889.037737:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235549312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176889.037752:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176889.037757:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11573) 00000800:00000200:1.1:1786176889.037793:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176889.037795:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176889.038017:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176889.038019:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176889.038201:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176889.038203:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176889.038529:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176889.038531:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.0:1786176889.484506:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786176889.484515:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786176889.484521:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000100:00000001:3.0:1786176889.548937:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176889.548947:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176889.548950:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176889.548955:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176889.548960:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176889.548964:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176889.548966:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176889.548969:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176889.548972:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176889.932473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176889.932479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176889.932487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176889.932492:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176889.932496:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176889.932501:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176889.932505:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176889.932508:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176889.932511:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176889.932514:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176889.932517:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176889.932517:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176889.932571:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176889.932575:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176889.932577:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176890.572846:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176890.572861:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176890.572865:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176890.572870:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176890.572877:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176890.572881:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176890.572884:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176890.572888:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176890.572893:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176890.956457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176890.956462:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176890.956471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176890.956475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176890.956483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176890.956486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176890.956538:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176890.956543:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176890.956547:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176891.596594:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176891.596605:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176891.596608:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176891.596615:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176891.596620:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176891.596624:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176891.596627:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176891.596630:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176891.596634:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176891.980455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176891.980462:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176891.980468:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176891.980475:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176891.980480:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176891.980483:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176891.980485:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176891.980488:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176891.980493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176891.980495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176891.980499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176891.980501:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176891.980503:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176891.980504:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176891.980506:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176891.980507:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176891.980509:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176891.980511:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176891.980603:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176891.980609:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176891.980613:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176892.620460:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176892.620466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176892.620468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176892.620473:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176892.620477:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176892.620481:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176892.620483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176892.620485:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176892.620489:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176892.940489:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306239744, at 4306239808 00000100:00000010:0.0:1786176892.940501:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62300. 00010000:00000001:0.0:1786176893.004449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176893.004455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176893.004462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176893.004465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176893.004468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176893.004469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176893.004511:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176893.004514:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176893.004518:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176893.644467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176893.644476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176893.644478:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176893.644485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176893.644489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176893.644494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176893.644497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176893.644499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176893.644502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176894.028464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176894.028471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176894.028476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176894.028478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176894.028485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176894.028487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176894.028542:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176894.028546:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176894.028549:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176894.156425:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176894.156433:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176894.156435:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176894.156438:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176894.156442:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176894.156449:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11568 00000400:00000200:3.0:1786176894.156456:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176894.156461:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176894.156463:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176894.156466:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156468:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176894.156470:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156472:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176894.156475:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176894.156476:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176894.156478:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176894.156481:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176894.156483:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176894.156484:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176894.156487:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156488:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176894.156489:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156491:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176894.156493:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176894.156495:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156497:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176894.156499:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176894.156501:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176894.156503:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.156504:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176894.156506:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176894.156513:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176894.156517:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176894.156520:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176894.156521:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.156524:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176894.156526:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176894.156530:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176894.156536:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176894.156541:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176894.156548:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235549440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176894.156556:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176894.156559:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176894.156563:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235549440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176894.156583:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176894.156586:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176894.156589:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176894.156590:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176894.156591:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176894.156593:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176894.156594:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11568 00000400:00000200:3.0:1786176894.156596:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176894.156598:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176894.156599:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176894.156601:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156603:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176894.156604:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156606:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176894.156608:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176894.156609:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176894.156611:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176894.156614:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176894.156616:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176894.156619:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176894.156623:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156626:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176894.156629:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156632:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176894.156637:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176894.156642:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156648:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176894.156652:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176894.156662:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176894.156666:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.156669:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176894.156674:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176894.156680:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.156683:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176894.156686:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176894.156688:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176894.156689:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176894.156690:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.156692:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176894.156696:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176894.156699:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176894.156706:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176894.156711:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176894.156719:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235549568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176894.156731:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176894.156735:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176894.156742:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235549568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176894.156762:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176894.156767:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176894.156775:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176894.156778:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176894.156781:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176894.156784:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176894.156790:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11568 00000400:00000200:3.0:1786176894.156798:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176894.156801:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176894.156804:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176894.156810:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156813:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176894.156816:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156820:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176894.156823:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176894.156824:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176894.156827:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176894.156830:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176894.156833:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176894.156837:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176894.156844:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156851:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176894.156860:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156867:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176894.156873:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176894.156882:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176894.156889:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176894.156894:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176894.156898:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176894.156901:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.156903:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176894.156907:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176894.156912:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.156914:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176894.156919:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000800:00000200:1.1:1786176894.156919:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176894.156925:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000020:00000040:3.0:1786176894.156933:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176894.156937:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176894.156940:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.156947:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176894.156950:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176894.156953:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176894.156959:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176894.156965:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176894.156975:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235549696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176894.156996:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176894.157000:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176894.157008:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235549696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176894.157023:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176894.157026:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176894.157031:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176894.157032:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176894.157033:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176894.157039:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176894.157043:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11568 00000400:00000200:3.0:1786176894.157049:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176894.157056:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176894.157059:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176894.157065:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.157068:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176894.157072:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.157077:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176894.157081:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176894.157083:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176894.157086:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176894.157088:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176894.157093:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176894.157096:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176894.157101:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.157104:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176894.157109:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.157115:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176894.157119:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176894.157125:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176894.157128:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176894.157133:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176894.157137:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176894.157141:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.157146:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176894.157151:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176894.157157:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.157163:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176894.157170:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176894.157177:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176894.157182:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176894.157186:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.157190:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176894.157194:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176894.157198:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176894.157204:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176894.157210:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176894.157218:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235549824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176894.157228:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176894.157232:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176894.157239:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235549824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176894.157254:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176894.157257:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176894.157274:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176894.157278:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176894.157280:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176894.157285:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176894.157290:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11568 00000400:00000200:3.0:1786176894.157296:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176894.157301:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176894.157305:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176894.157312:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.157316:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176894.157320:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.157325:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176894.157329:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176894.157330:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176894.157332:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176894.157336:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176894.157340:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176894.157346:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176894.157384:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.157392:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176894.157397:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176894.157403:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176894.157408:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176894.157412:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176894.157416:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176894.157419:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176894.157424:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176894.157430:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.157436:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176894.157438:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176894.157441:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.157444:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176894.157449:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176894.157453:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176894.157456:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176894.157459:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176894.157462:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176894.157463:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176894.157466:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176894.157472:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176894.157478:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176894.157485:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235549952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176894.157494:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176894.157498:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176894.157504:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235549952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176894.157513:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176894.157519:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11578) 00000800:00000200:1.1:1786176894.157593:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176894.157594:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176894.157788:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176894.157790:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176894.157960:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176894.157962:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176894.158220:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176894.158223:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1786176894.668475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176894.668483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176894.668487:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176894.668493:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176894.668497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176894.668500:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176894.668506:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176894.668509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176894.668512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176895.052468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176895.052478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176895.052488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176895.052490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176895.052493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176895.052493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176895.052564:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176895.052569:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176895.052573:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176895.692440:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176895.692449:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176895.692452:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176895.692458:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176895.692463:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176895.692467:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176895.692470:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176895.692473:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176895.692476:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176896.076486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176896.076491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176896.076497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176896.076500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176896.076504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176896.076506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176896.076577:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176896.076582:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176896.076586:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176896.716458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176896.716466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176896.716469:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176896.716475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176896.716479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176896.716484:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176896.716487:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176896.716490:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176896.716494:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176897.100472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176897.100477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176897.100483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176897.100486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176897.100491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176897.100493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176897.100553:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176897.100558:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176897.100561:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176897.740472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176897.740480:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176897.740483:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176897.740489:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176897.740494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176897.740498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176897.740501:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176897.740505:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176897.740508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176898.060448:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306244864, at 4306244928 00000100:00000010:0.0:1786176898.060467:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62a20. 00010000:00000001:0.0:1786176898.124461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176898.124465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176898.124472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176898.124475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176898.124480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176898.124483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176898.124541:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176898.124545:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176898.124547:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176898.764463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176898.764473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176898.764476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176898.764482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176898.764487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176898.764491:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176898.764494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176898.764496:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176898.764500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176899.148513:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176899.148518:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176899.148526:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176899.148529:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176899.148534:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176899.148536:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176899.148613:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176899.148620:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176899.148625:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786176899.276530:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176899.276541:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786176899.276544:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176899.276548:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176899.276553:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176899.276561:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11573 00000400:00000200:2.0:1786176899.276571:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176899.276578:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176899.276581:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176899.276585:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.276588:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176899.276591:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.276594:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176899.276599:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176899.276601:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786176899.276604:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176899.276607:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786176899.276610:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786176899.276612:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786176899.276616:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.276618:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176899.276620:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.276622:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176899.276625:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786176899.276629:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176899.276632:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176899.276634:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176899.276638:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176899.276652:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.276657:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176899.276659:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176899.276670:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786176899.276674:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786176899.276678:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176899.276681:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.276684:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176899.276687:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176899.276691:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176899.276697:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aabc00. 00000100:00000001:2.0:1786176899.276705:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176899.276716:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235550080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176899.276726:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176899.276731:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176899.276736:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235550080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176899.276763:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176899.276767:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176899.276770:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786176899.276772:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176899.276774:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176899.276777:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176899.276780:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11573 00000400:00000200:2.0:1786176899.276785:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176899.276788:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176899.276791:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176899.276795:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.276798:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176899.276800:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.276803:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176899.276807:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176899.276809:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786176899.276811:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176899.276813:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786176899.276816:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786176899.276819:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786176899.276823:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.276825:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176899.276828:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.276831:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176899.276836:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786176899.276844:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176899.276852:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176899.276857:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176899.276863:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176899.276868:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.276872:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176899.276878:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176899.276882:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.276885:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176899.276888:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 00000020:00000040:2.0:1786176899.276895:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176899.276904:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176899.276910:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.276913:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176899.276917:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176899.276921:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176899.276930:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:2.0:1786176899.276938:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176899.276953:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235550208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176899.276969:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176899.276978:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176899.276988:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235550208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176899.277023:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176899.277030:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176899.277039:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786176899.277041:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176899.277043:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176899.277050:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176899.277059:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11573 00000400:00000200:2.0:1786176899.277069:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176899.277079:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176899.277090:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176899.277100:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277112:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176899.277121:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277129:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176899.277135:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176899.277136:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786176899.277138:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176899.277140:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786176899.277142:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786176899.277149:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786176899.277159:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277162:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176899.277168:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277175:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176899.277178:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786176899.277190:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277196:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176899.277203:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176899.277224:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176899.277233:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.277240:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176899.277245:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176899.277256:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.277265:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176899.277274:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000020:00000040:2.0:1786176899.277279:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176899.277284:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176899.277289:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.277296:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176899.277304:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176899.277311:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176899.277319:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9000. 00000100:00000001:2.0:1786176899.277325:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176899.277337:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235550336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176899.277412:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1786176899.277415:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786176899.277419:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176899.277424:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000040:2.0:1786176899.277431:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235550336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176899.277454:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176899.277459:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176899.277466:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786176899.277470:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176899.277475:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176899.277486:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176899.277493:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11573 00000400:00000200:2.0:1786176899.277503:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176899.277512:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176899.277518:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176899.277528:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277534:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176899.277539:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277546:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176899.277554:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176899.277557:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786176899.277562:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176899.277567:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786176899.277572:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786176899.277580:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786176899.277594:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277603:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176899.277609:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277617:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176899.277626:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786176899.277634:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277637:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176899.277641:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176899.277646:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176899.277655:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.277665:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176899.277668:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176899.277675:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.277680:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176899.277686:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000020:00000040:2.0:1786176899.277691:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176899.277695:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176899.277697:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.277704:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176899.277708:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176899.277712:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176899.277719:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:2.0:1786176899.277727:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176899.277741:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235550464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176899.277764:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176899.277768:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176899.277779:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235550464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176899.277809:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176899.277815:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176899.277822:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786176899.277828:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176899.277833:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176899.277842:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176899.277850:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11573 00000400:00000200:2.0:1786176899.277864:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176899.277874:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176899.277880:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176899.277889:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277900:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176899.277910:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277919:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176899.277924:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176899.277929:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786176899.277934:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176899.277940:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786176899.277948:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786176899.277955:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786176899.277968:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277981:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176899.277990:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176899.277995:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176899.278005:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786176899.278020:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176899.278027:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176899.278034:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176899.278040:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176899.278047:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.278052:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176899.278057:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176899.278064:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.278070:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176899.278077:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 00000020:00000040:2.0:1786176899.278084:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176899.278092:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176899.278096:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176899.278102:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176899.278109:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176899.278114:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176899.278123:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786176899.278137:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176899.278153:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235550592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176899.278224:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176899.278234:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176899.278249:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235550592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176899.278277:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176899.278287:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11583) 00000800:00000200:1.1:1786176899.278502:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176899.278505:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176899.278837:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176899.278840:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176899.279163:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176899.279176:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176899.279535:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176899.279538:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1786176899.788521:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176899.788528:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176899.788531:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176899.788536:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176899.788540:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176899.788543:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176899.788547:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176899.788549:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176899.788552:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176900.172489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176900.172495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176900.172503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176900.172507:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176900.172511:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176900.172516:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176900.172521:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176900.172524:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176900.172526:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176900.172531:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176900.172534:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176900.172538:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176900.172598:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176900.172603:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176900.172607:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176900.812488:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176900.812495:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176900.812497:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176900.812502:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176900.812505:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176900.812508:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176900.812510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176900.812512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176900.812514:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176901.196517:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176901.196524:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176901.196532:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176901.196536:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176901.196544:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176901.196547:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176901.196606:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176901.196613:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176901.196618:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176901.836483:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176901.836491:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176901.836496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176901.836502:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176901.836506:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176901.836510:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176901.836514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176901.836516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176901.836519:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176902.220462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176902.220471:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176902.220474:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176902.220479:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176902.220484:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176902.220488:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176902.220491:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176902.220494:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176902.220498:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176902.220500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176902.220502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176902.220504:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176902.220507:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176902.220510:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176902.220513:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176902.220514:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176902.220520:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176902.220522:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176902.860431:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176902.860439:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176902.860441:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176902.860447:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176902.860451:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176902.860455:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176902.860457:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176902.860459:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176902.860462:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176903.180476:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306249985, at 4306250048 00000100:00000010:0.0:1786176903.180488:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62360. 00010000:00000001:0.0:1786176903.244485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176903.244491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176903.244499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176903.244503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176903.244509:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176903.244514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786176903.244583:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786176903.244588:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786176903.244593:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176903.884470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176903.884477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176903.884480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176903.884484:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176903.884488:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176903.884492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176903.884494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176903.884496:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176903.884498:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176904.268436:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176904.268442:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176904.268448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176904.268452:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176904.268458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176904.268461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176904.268518:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176904.268521:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176904.268524:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176904.396499:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176904.396507:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176904.396509:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176904.396513:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176904.396516:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176904.396522:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11578 00000400:00000200:3.0:1786176904.396529:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176904.396534:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176904.396536:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176904.396539:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396541:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176904.396543:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396545:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176904.396549:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176904.396551:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176904.396553:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176904.396556:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176904.396558:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176904.396559:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176904.396561:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396562:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176904.396564:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396565:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176904.396568:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176904.396571:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396574:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176904.396577:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176904.396580:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176904.396583:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.396586:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176904.396588:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176904.396598:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176904.396602:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176904.396606:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176904.396609:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.396612:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176904.396615:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176904.396619:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176904.396626:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176904.396631:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176904.396638:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235550720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176904.396646:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176904.396650:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176904.396653:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235550720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176904.396673:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176904.396675:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176904.396678:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176904.396679:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176904.396681:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176904.396682:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176904.396684:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11578 00000400:00000200:3.0:1786176904.396687:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176904.396689:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176904.396691:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176904.396693:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396695:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176904.396697:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396699:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176904.396701:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176904.396702:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176904.396703:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176904.396704:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176904.396706:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176904.396707:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176904.396709:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396711:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176904.396712:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396713:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176904.396716:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176904.396719:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396722:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176904.396724:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176904.396730:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176904.396733:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.396735:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176904.396739:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176904.396743:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.396746:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176904.396749:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176904.396750:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176904.396752:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176904.396753:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.396754:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176904.396756:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176904.396759:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176904.396763:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176904.396768:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176904.396776:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235550848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176904.396784:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176904.396788:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176904.396793:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235550848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176904.396804:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176904.396807:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176904.396814:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176904.396816:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176904.396817:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176904.396819:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176904.396823:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11578 00000400:00000200:3.0:1786176904.396828:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176904.396833:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176904.396838:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176904.396842:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396844:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176904.396847:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396852:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176904.396854:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176904.396855:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176904.396856:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176904.396857:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176904.396861:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176904.396865:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176904.396872:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396876:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176904.396880:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396883:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176904.396887:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176904.396891:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176904.396895:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176904.396898:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176904.396901:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176904.396904:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.396906:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176904.396910:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176904.396916:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.396917:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176904.396921:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176904.396925:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176904.396927:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176904.396928:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.396929:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176904.396931:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176904.396933:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176904.396937:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176904.396941:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176904.396948:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235550976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176904.396958:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176904.396959:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176904.396963:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235550976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176904.396976:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176904.396980:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176904.396989:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176904.396992:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176904.396994:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176904.397001:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176904.397008:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11578 00000400:00000200:3.0:1786176904.397018:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176904.397026:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176904.397029:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176904.397034:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.397040:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176904.397045:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:1.1:1786176904.397049:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176904.397050:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176904.397056:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176904.397058:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176904.397059:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176904.397061:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176904.397063:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176904.397064:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176904.397067:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.397068:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176904.397071:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.397072:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176904.397078:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176904.397082:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176904.397085:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176904.397089:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176904.397092:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176904.397096:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.397098:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176904.397100:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176904.397105:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.397107:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176904.397111:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176904.397114:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176904.397117:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176904.397118:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.397120:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176904.397121:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176904.397123:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176904.397128:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176904.397132:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176904.397139:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235551104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176904.397149:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176904.397152:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176904.397156:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235551104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176904.397183:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176904.397188:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176904.397193:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176904.397194:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176904.397198:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176904.397201:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176904.397205:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11578 00000400:00000200:3.0:1786176904.397210:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176904.397214:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176904.397218:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176904.397223:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.397226:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176904.397229:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.397232:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176904.397238:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176904.397240:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176904.397243:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176904.397245:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176904.397249:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176904.397255:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176904.397264:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.397270:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176904.397276:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176904.397280:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176904.397287:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176904.397297:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176904.397304:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176904.397306:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176904.397309:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176904.397312:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.397316:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176904.397320:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176904.397325:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.397328:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176904.397335:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176904.397340:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176904.397345:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176904.397349:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176904.397352:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176904.397355:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176904.397388:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176904.397395:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176904.397402:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176904.397412:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235551232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176904.397427:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176904.397433:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176904.397441:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235551232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176904.397466:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176904.397472:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11588) 00000800:00000200:1.1:1786176904.397495:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176904.397657:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176904.397804:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176904.397963:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176904.908524:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176904.908533:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176904.908536:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176904.908542:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176904.908546:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176904.908551:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176904.908554:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176904.908557:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176904.908565:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176905.292506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176905.292517:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176905.292527:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176905.292530:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176905.292537:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176905.292540:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176905.932456:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176905.932462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176905.932464:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176905.932469:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176905.932473:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176905.932478:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176905.932481:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176905.932484:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176905.932488:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176906.316458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176906.316464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176906.316470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176906.316474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176906.316478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176906.316479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176906.316526:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176906.316530:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176906.316534:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176906.956446:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176906.956455:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176906.956457:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176906.956463:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176906.956467:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176906.956470:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176906.956473:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176906.956476:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176906.956479:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176907.340490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176907.340497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176907.340504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176907.340507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176907.340512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176907.340513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176907.340601:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176907.340607:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176907.340611:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1786176907.468713:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:0.0:1786176907.468763:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1786176907.468775:0:2383:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176907.468780:0:2383:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:0.0:1786176907.468786:0:2383:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176907.468792:0:2383:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1786176907.468799:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176907.468804:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:0.0:1786176907.468811:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176907.468817:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176907.468825:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e110 00000400:00000010:0.0:1786176907.468829:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e110. 00000100:00000001:0.0:1786176907.468835:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176907.468847:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26324700 x1872930235550720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176907.468884:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176907.468890:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324700 x1872930235550720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176907.468898:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176907.468900:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176907.468913:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176907.468915:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:0.0:1786176907.468918:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176907.468922:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176907.468926:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e770 00000400:00000010:0.0:1786176907.468928:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e770. 00000100:00000001:0.0:1786176907.468931:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176907.468935:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26327b80 x1872930235550848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176907.468944:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176907.468947:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327b80 x1872930235550848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176907.468953:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176907.468955:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176907.468959:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176907.468961:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:0.0:1786176907.468964:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176907.468967:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176907.468970:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e990 00000400:00000010:0.0:1786176907.468972:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e990. 00000100:00000001:0.0:1786176907.468974:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176907.468979:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26325880 x1872930235550976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176907.468985:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176907.468988:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235550976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176907.468993:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176907.468995:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176907.469000:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176907.469002:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:0.0:1786176907.469005:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176907.469008:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176907.469010:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eb28 00000400:00000010:0.0:1786176907.469012:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eb28. 00000100:00000001:0.0:1786176907.469014:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176907.469018:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26325500 x1872930235551104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176907.469026:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176907.469029:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235551104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176907.469042:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176907.469045:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786176907.469052:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786176907.469054:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:0.0:1786176907.469058:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786176907.469062:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786176907.469066:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e440 00000400:00000010:0.0:1786176907.469069:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e440. 00000100:00000001:0.0:1786176907.469077:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786176907.469085:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26324a80 x1872930235551232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176907.469098:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786176907.469104:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324a80 x1872930235551232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176929 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176907.469146:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176907.469150:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1786176907.980457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176907.980465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176907.980469:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176907.980475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176907.980480:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176907.980485:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176907.980488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176907.980492:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176907.980496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176908.300470:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306255104, at 4306255168 00000100:00000010:0.0:1786176908.300484:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62f00. 00010000:00000001:0.0:1786176908.364440:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176908.364446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176908.364451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176908.364453:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176908.364457:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176908.364458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176908.364516:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176908.364519:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176908.364522:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176909.004465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176909.004474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176909.004477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176909.004483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176909.004487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176909.004491:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176909.004494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176909.004497:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176909.004500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176909.388460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176909.388465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176909.388473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176909.388477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176909.388485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176909.388490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176909.388560:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176909.388564:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176909.388567:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176909.516398:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176909.516407:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176909.516409:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176909.516412:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176909.516416:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176909.516422:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11583 00000400:00000200:3.0:1786176909.516429:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176909.516434:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176909.516437:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176909.516441:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516444:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176909.516446:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516448:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176909.516452:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176909.516453:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176909.516455:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176909.516458:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176909.516459:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176909.516461:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176909.516464:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516466:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176909.516467:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516468:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176909.516470:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176909.516473:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516474:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176909.516476:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176909.516478:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176909.516480:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.516482:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176909.516483:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176909.516488:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786176909.516491:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176909.516493:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176909.516495:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.516497:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176909.516499:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176909.516502:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176909.516507:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176909.516513:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176909.516520:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235551360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176909.516528:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176909.516531:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176909.516536:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235551360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176909.516561:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176909.516565:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176909.516569:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176909.516570:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176909.516572:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176909.516575:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176909.516578:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11583 00000400:00000200:3.0:1786176909.516583:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176909.516587:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176909.516590:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176909.516594:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516598:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176909.516601:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516604:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176909.516607:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176909.516608:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176909.516610:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176909.516612:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176909.516616:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176909.516620:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176909.516626:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516629:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176909.516633:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516637:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176909.516642:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176909.516647:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516650:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176909.516655:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176909.516661:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176909.516665:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.516668:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176909.516674:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176909.516677:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.516688:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176909.516694:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786176909.516698:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176909.516702:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176909.516704:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.516706:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176909.516707:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176909.516709:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176909.516716:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176909.516719:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176909.516727:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235551488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176909.516741:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176909.516745:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176909.516755:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235551488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176909.516774:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176909.516778:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176909.516784:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176909.516786:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176909.516787:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176909.516792:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176909.516809:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11583 00000400:00000200:3.0:1786176909.516819:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176909.516826:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176909.516830:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176909.516832:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516835:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176909.516841:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516847:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176909.516853:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176909.516856:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176909.516859:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176909.516862:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176909.516866:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176909.516879:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176909.516886:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516893:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176909.516900:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516903:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176909.516909:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176909.516915:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176909.516920:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176909.516923:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176909.516927:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176909.516930:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.516932:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176909.516937:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176909.516942:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.516945:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176909.516952:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000800:00000200:1.1:1786176909.516955:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786176909.516957:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000800:00000200:1.1:1786176909.516961:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 02000000:00000001:3.0:1786176909.516963:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176909.516967:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.516971:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176909.516974:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176909.516978:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176909.516985:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176909.516992:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176909.517002:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235551616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176909.517016:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176909.517027:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176909.517036:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235551616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176909.517051:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176909.517056:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176909.517065:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176909.517068:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176909.517070:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176909.517073:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176909.517078:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11583 00000400:00000200:3.0:1786176909.517085:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176909.517091:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176909.517096:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176909.517102:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.517107:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176909.517131:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.517137:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176909.517140:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176909.517142:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176909.517146:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176909.517150:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176909.517153:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176909.517159:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176909.517166:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.517171:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176909.517176:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.517181:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176909.517186:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176909.517193:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176909.517199:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176909.517203:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176909.517209:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176909.517213:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.517217:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176909.517222:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176909.517227:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.517232:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176909.517240:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176909.517245:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176909.517250:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176909.517253:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.517257:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176909.517259:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176909.517262:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176909.517270:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176909.517276:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176909.517286:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235551744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176909.517303:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176909.517308:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176909.517318:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235551744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176909.517334:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176909.517341:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176909.517347:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176909.517351:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176909.517354:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1786176909.517614:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176909.517617:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176909.517927:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176909.517929:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786176909.518038:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176909.518046:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11583 00000400:00000200:3.0:1786176909.518055:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176909.518063:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176909.518067:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176909.518074:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.518080:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176909.518086:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.518091:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176909.518097:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176909.518100:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176909.518104:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176909.518108:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176909.518129:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176909.518135:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176909.518144:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.518149:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176909.518154:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176909.518158:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176909.518162:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176909.518169:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176909.518175:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176909.518180:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176909.518186:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176909.518190:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.518195:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176909.518199:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176909.518206:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.518210:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176909.518217:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176909.518224:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176909.518229:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176909.518232:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176909.518236:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176909.518239:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176909.518243:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176909.518250:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176909.518257:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176909.518268:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235551872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176909.518285:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176909.518291:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176909.518300:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235551872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176909.518318:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176909.518325:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11594) 00000800:00000200:1.1:1786176909.518443:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176909.518446:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176909.518858:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176909.518861:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786176910.028468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176910.028477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176910.028480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176910.028486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176910.028491:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176910.028495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176910.028498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176910.028500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176910.028502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176910.412442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176910.412445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176910.412451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176910.412455:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176910.412457:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176910.412461:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176910.412463:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176910.412466:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176910.412468:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176910.412470:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176910.412474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176910.412475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176910.412544:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176910.412549:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176910.412552:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176911.052532:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176911.052545:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176911.052548:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176911.052557:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176911.052563:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176911.052569:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176911.052573:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176911.052576:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176911.052580:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176911.436468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176911.436477:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176911.436480:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176911.436484:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176911.436490:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176911.436493:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176911.436497:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176911.436504:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176911.436515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176911.436518:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176911.436520:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176911.436523:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176911.436524:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176911.436526:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176911.436530:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176911.436532:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176911.436539:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176911.436545:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176911.436603:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176911.436607:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176911.436610:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176912.076624:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176912.076630:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176912.076632:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176912.076638:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176912.076641:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176912.076645:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176912.076648:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176912.076650:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176912.076653:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176912.460471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176912.460477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176912.460483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176912.460485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176912.460490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176912.460492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176912.460563:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176912.460569:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176912.460571:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176913.100496:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176913.100502:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176913.100504:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176913.100509:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176913.100511:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176913.100514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176913.100516:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176913.100518:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176913.100520:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176913.420445:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306260224, at 4306260288 00000100:00000010:0.0:1786176913.420458:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62780. 00010000:00000001:0.0:1786176913.484461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176913.484466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176913.484476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176913.484479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176913.484485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176913.484490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176913.484565:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176913.484570:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176913.484574:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176914.124506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176914.124514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176914.124516:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176914.124522:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176914.124526:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176914.124531:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176914.124534:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176914.124537:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176914.124540:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176914.508519:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176914.508530:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176914.508540:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176914.508548:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176914.508557:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176914.508562:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176914.508623:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176914.508629:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176914.508632:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176914.636498:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176914.636509:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176914.636512:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176914.636516:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176914.636520:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176914.636529:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11589 00000400:00000200:3.0:1786176914.636538:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176914.636545:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176914.636548:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176914.636550:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.636554:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176914.636556:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.636558:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176914.636562:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176914.636564:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176914.636567:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176914.636570:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176914.636571:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176914.636573:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176914.636575:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.636577:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176914.636579:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.636580:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176914.636583:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176914.636586:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176914.636587:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176914.636589:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176914.636593:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176914.636596:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.636598:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176914.636600:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176914.636613:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176914.636617:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176914.636621:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176914.636623:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.636627:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176914.636631:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176914.636637:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176914.636647:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176914.636656:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176914.636670:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235552000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176914.636682:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176914.636688:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176914.636694:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235552000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176914.636722:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176914.636725:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176914.636732:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176914.636733:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176914.636735:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176914.636737:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176914.636740:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11589 00000400:00000200:3.0:1786176914.636743:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176914.636746:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176914.636748:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176914.636751:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.636752:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176914.636754:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.636755:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176914.636757:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176914.636758:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176914.636759:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176914.636761:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176914.636763:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176914.636765:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176914.636769:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.636771:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176914.636772:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.636774:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176914.636776:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176914.636778:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176914.636780:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176914.636781:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176914.636788:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176914.636791:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.636793:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176914.636805:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176914.636815:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.636821:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176914.636825:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176914.636828:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176914.636834:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176914.636839:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.636843:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176914.636848:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176914.636852:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176914.636862:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176914.636871:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176914.636884:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235552128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176914.636896:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176914.636902:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176914.636926:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235552128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176914.636957:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176914.636965:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176914.636973:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176914.636981:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176914.636986:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176914.636993:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176914.637001:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11589 00000400:00000200:3.0:1786176914.637010:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176914.637017:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176914.637022:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176914.637029:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637036:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176914.637045:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637051:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176914.637056:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176914.637058:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176914.637079:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176914.637081:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176914.637083:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176914.637086:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176914.637090:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637097:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176914.637103:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637115:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176914.637126:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176914.637137:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637146:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176914.637151:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176914.637158:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176914.637164:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.637167:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176914.637172:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176914.637175:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.637176:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176914.637181:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176914.637183:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176914.637192:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176914.637194:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.637196:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176914.637201:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176914.637205:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176914.637213:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176914.637217:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176914.637227:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235552256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176914.637255:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176914.637260:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176914.637275:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235552256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176914.637314:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176914.637320:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176914.637328:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176914.637336:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176914.637342:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176914.637347:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176914.637386:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11589 00000400:00000200:3.0:1786176914.637391:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176914.637395:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176914.637397:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1786176914.637400:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176914.637407:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000040:3.0:1786176914.637411:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637417:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176914.637420:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637429:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176914.637441:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176914.637445:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176914.637455:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176914.637462:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176914.637468:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176914.637478:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176914.637490:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637497:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176914.637502:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637509:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176914.637516:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176914.637522:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637525:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176914.637528:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176914.637532:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176914.637536:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.637537:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176914.637543:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176914.637556:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.637564:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176914.637573:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176914.637580:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176914.637588:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176914.637590:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.637592:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176914.637593:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176914.637600:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176914.637607:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176914.637613:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176914.637621:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235552384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176914.637629:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176914.637631:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176914.637640:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235552384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176914.637658:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176914.637667:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176914.637681:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176914.637692:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176914.637699:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176914.637707:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176914.637711:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11589 00000400:00000200:3.0:1786176914.637724:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176914.637737:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176914.637747:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176914.637761:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637768:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176914.637775:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637787:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176914.637792:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176914.637796:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176914.637803:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176914.637808:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176914.637820:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176914.637826:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176914.637838:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637846:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176914.637856:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637862:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176914.637867:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176914.637874:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176914.637879:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176914.637884:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176914.637889:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176914.637898:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.637911:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176914.637919:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176914.637923:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.637926:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176914.637937:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176914.637945:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176914.637954:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176914.637961:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176914.637971:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176914.637977:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176914.637988:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176914.638000:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176914.638011:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176914.638022:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235552512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176914.638104:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176914.638109:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176914.638123:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235552512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176914.638157:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176914.638164:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11599) 00000800:00000200:1.1:1786176914.638285:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176914.638287:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176914.638550:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176914.638552:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176914.638738:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176914.638740:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176914.638920:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176914.638921:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1786176915.148501:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176915.148517:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176915.148520:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176915.148529:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176915.148535:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176915.148541:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176915.148544:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176915.148547:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176915.148550:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176915.532516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176915.532523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176915.532528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176915.532533:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176915.532538:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176915.532544:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176915.532619:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176915.532624:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176915.532628:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176916.172499:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176916.172517:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176916.172521:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176916.172530:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176916.172537:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176916.172542:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176916.172546:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176916.172550:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176916.172554:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176916.556475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176916.556481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176916.556487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176916.556490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176916.556496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176916.556499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176916.556584:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176916.556587:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176916.556590:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176917.196478:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176917.196485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176917.196488:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176917.196495:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176917.196500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176917.196505:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176917.196508:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176917.196511:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176917.196514:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176917.580436:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176917.580441:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176917.580448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176917.580451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176917.580455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176917.580459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176917.580548:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176917.580552:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176917.580555:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176918.220469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176918.220477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176918.220480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176918.220486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176918.220490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176918.220493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176918.220496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176918.220499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176918.220502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176918.540489:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306265345, at 4306265408 00000100:00000010:0.0:1786176918.540505:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62480. 00010000:00000001:0.0:1786176918.604469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176918.604474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176918.604480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176918.604483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176918.604490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176918.604492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176918.604560:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176918.604565:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176918.604568:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176919.244465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176919.244474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176919.244477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176919.244483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176919.244489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176919.244494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176919.244497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176919.244502:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176919.244508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176919.628455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176919.628459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176919.628464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176919.628466:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176919.628468:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176919.628471:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176919.628475:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176919.628478:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176919.628482:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176919.628485:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176919.628490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176919.628494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176919.628558:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176919.628563:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176919.628567:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786176919.756505:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176919.756517:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786176919.756520:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176919.756524:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176919.756528:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176919.756546:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11594 00000400:00000200:2.0:1786176919.756556:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176919.756563:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176919.756566:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176919.756570:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.756573:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176919.756576:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.756579:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176919.756583:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176919.756585:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786176919.756588:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176919.756592:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786176919.756594:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786176919.756598:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786176919.756601:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.756603:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176919.756605:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.756615:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176919.756619:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786176919.756622:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176919.756625:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176919.756628:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176919.756632:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176919.756635:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.756637:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176919.756640:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176919.756646:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 00000020:00000040:2.0:1786176919.756651:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786176919.756655:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176919.756656:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.756660:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176919.756663:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176919.756667:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176919.756675:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000001:2.0:1786176919.756682:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176919.756692:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235552640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176919.756714:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176919.756719:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176919.756724:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235552640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176919.756746:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176919.756750:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176919.756753:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786176919.756755:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176919.756758:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176919.756761:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176919.756764:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11594 00000400:00000200:2.0:1786176919.756769:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176919.756773:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176919.756776:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176919.756783:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.756789:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176919.756795:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.756800:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176919.756806:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176919.756809:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786176919.756815:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176919.756821:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786176919.756828:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786176919.756834:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786176919.756842:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.756849:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176919.756856:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.756863:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176919.756868:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786176919.756876:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176919.756882:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176919.756886:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176919.756893:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176919.756899:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.756904:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176919.756909:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176919.756916:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.756921:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176919.756929:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000020:00000040:2.0:1786176919.756937:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176919.756942:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176919.756946:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.756949:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176919.756953:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176919.756955:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176919.756963:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:2.0:1786176919.756970:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176919.756982:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235552768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176919.757002:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176919.757009:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176919.757032:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235552768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176919.757064:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176919.757072:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176919.757081:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.1:1786176919.757086:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786176919.757087:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176919.757090:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1786176919.757093:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1786176919.757095:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176919.757102:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11594 00000400:00000200:2.0:1786176919.757111:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176919.757118:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176919.757123:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176919.757130:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757135:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176919.757142:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757147:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176919.757152:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176919.757156:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786176919.757159:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176919.757164:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786176919.757169:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786176919.757174:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786176919.757181:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757195:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176919.757202:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757207:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176919.757214:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786176919.757222:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757229:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176919.757232:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176919.757239:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176919.757244:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.757249:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176919.757253:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176919.757259:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.757262:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176919.757267:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000020:00000040:2.0:1786176919.757272:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176919.757278:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176919.757281:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.757287:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176919.757290:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176919.757294:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176919.757301:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786176919.757307:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176919.757318:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235552896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176919.757336:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176919.757342:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176919.757354:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235552896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176919.757415:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176919.757422:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176919.757433:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786176919.757436:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176919.757439:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176919.757443:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176919.757450:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11594 00000400:00000200:2.0:1786176919.757461:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176919.757470:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176919.757475:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176919.757483:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757489:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176919.757493:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757500:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176919.757504:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176919.757507:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786176919.757511:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176919.757514:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786176919.757520:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786176919.757526:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786176919.757536:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757542:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176919.757549:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757556:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176919.757563:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786176919.757572:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757577:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176919.757582:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176919.757587:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176919.757593:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.757598:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176919.757602:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176919.757609:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.757614:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176919.757622:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000020:00000040:2.0:1786176919.757628:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176919.757635:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176919.757639:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.757646:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176919.757649:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176919.757654:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176919.757662:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:2.0:1786176919.757668:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176919.757678:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235553024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176919.757697:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176919.757701:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176919.757711:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235553024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176919.757729:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176919.757734:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176919.757745:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786176919.757749:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176919.757754:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176919.757759:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176919.757765:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11594 00000400:00000200:2.0:1786176919.757776:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176919.757787:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176919.757793:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176919.757804:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757811:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176919.757818:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757824:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176919.757830:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176919.757834:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786176919.757838:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176919.757842:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786176919.757848:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786176919.757854:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000800:00000200:1.1:1786176919.757858:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176919.757862:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000040:2.0:1786176919.757863:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757870:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176919.757877:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757883:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176919.757891:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786176919.757901:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176919.757907:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176919.757912:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176919.757916:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176919.757921:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.757925:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176919.757929:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176919.757934:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.757939:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176919.757945:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000020:00000040:2.0:1786176919.757950:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176919.757956:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176919.757960:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176919.757963:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176919.757967:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176919.757970:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176919.757979:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:2.0:1786176919.757987:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176919.758000:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235553152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176919.758045:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176919.758051:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176919.758061:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235553152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176919.758094:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176919.758102:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11604) 00000800:00000200:1.1:1786176919.758284:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176919.758287:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176919.758577:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176919.758579:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176919.758828:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176919.758830:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:0.0:1786176920.268443:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176920.268452:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176920.268455:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176920.268462:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176920.268466:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176920.268470:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176920.268473:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176920.268476:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176920.268479:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176920.652463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176920.652468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176920.652476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176920.652479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176920.652483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176920.652485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176920.652544:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176920.652549:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176920.652552:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176921.292500:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176921.292506:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176921.292508:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176921.292513:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176921.292517:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176921.292521:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176921.292523:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176921.292525:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176921.292528:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176921.676451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176921.676458:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176921.676462:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176921.676466:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176921.676470:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176921.676478:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176921.676482:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176921.676485:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176921.676490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176921.676493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176921.676495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176921.676497:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176921.676498:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176921.676499:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176921.676501:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176921.676502:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176921.676503:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176921.676504:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176921.676551:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176921.676556:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176921.676560:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176922.316460:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176922.316471:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176922.316474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176922.316483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176922.316489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176922.316494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176922.316498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176922.316503:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176922.316508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176922.700450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176922.700456:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176922.700466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176922.700470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176922.700477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176922.700481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176922.700544:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176922.700549:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176922.700554:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176923.340489:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176923.340500:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176923.340511:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176923.340518:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176923.340523:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176923.340527:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176923.340530:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176923.340533:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176923.340536:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176923.660504:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306270464, at 4306270528 00000100:00000010:0.0:1786176923.660526:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62d20. 00010000:00000001:0.0:1786176923.724443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176923.724448:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176923.724455:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176923.724457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176923.724460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176923.724463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176923.724507:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176923.724514:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176923.724518:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176924.748501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176924.748506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176924.748515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176924.748518:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176924.748524:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176924.748528:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176924.748592:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176924.748596:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176924.748599:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786176924.876445:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176924.876456:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786176924.876459:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176924.876463:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176924.876468:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176924.876476:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11599 00000400:00000200:2.0:1786176924.876484:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176924.876490:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176924.876493:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176924.876497:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876500:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176924.876502:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876504:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176924.876508:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176924.876509:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786176924.876512:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176924.876515:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786176924.876517:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786176924.876519:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786176924.876521:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876522:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176924.876523:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876525:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176924.876527:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786176924.876537:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876539:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176924.876541:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176924.876545:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176924.876547:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.876549:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176924.876551:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176924.876562:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734e00. 00000020:00000040:2.0:1786176924.876565:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786176924.876569:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176924.876570:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.876573:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176924.876576:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176924.876581:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176924.876590:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:2.0:1786176924.876596:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176924.876607:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734e00 x1872930235553280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176924.876616:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176924.876621:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176924.876625:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001f62212] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734e00 x1872930235553280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176924.876645:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176924.876647:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176924.876650:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786176924.876652:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176924.876653:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176924.876656:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176924.876659:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11599 00000400:00000200:2.0:1786176924.876662:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176924.876665:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176924.876667:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176924.876669:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876670:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176924.876673:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876675:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176924.876678:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176924.876679:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786176924.876681:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176924.876685:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786176924.876687:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786176924.876691:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786176924.876694:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876696:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176924.876701:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876705:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176924.876710:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786176924.876716:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876722:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176924.876725:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176924.876732:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176924.876736:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.876739:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176924.876744:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176924.876748:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.876752:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176924.876756:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735180. 00000020:00000040:2.0:1786176924.876760:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176924.876764:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176924.876766:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.876769:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176924.876772:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176924.876775:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176924.876779:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab400. 00000100:00000001:2.0:1786176924.876783:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176924.876790:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735180 x1872930235553408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176924.876800:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176924.876804:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176924.876808:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ba0f078] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735180 x1872930235553408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176924.876817:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176924.876820:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176924.876822:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786176924.876824:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176924.876827:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176924.876831:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176924.876834:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11599 00000400:00000200:2.0:1786176924.876837:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176924.876843:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176924.876847:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176924.876852:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876854:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176924.876858:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876863:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176924.876868:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176924.876871:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786176924.876874:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176924.876878:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786176924.876888:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786176924.876891:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786176924.876896:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876898:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176924.876902:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876904:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176924.876905:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786176924.876907:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176924.876909:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176924.876912:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176924.876916:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176924.876918:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.876919:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176924.876923:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176924.876926:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.876928:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176924.876933:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000020:00000040:2.0:1786176924.876938:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176924.876942:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176924.876947:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.876951:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176924.876955:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176924.876959:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176924.876981:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786176924.876990:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176924.876999:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235553536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176924.877012:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176924.877017:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176924.877022:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235553536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176924.877029:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176924.877034:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176924.877039:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786176924.877043:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176924.877045:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1786176924.877050:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786176924.877051:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1786176924.877056:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000100:00080000:2.0:1786176924.877057:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11599 00000400:00000200:2.0:1786176924.877065:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176924.877071:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176924.877075:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176924.877077:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.877080:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176924.877084:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.877089:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176924.877094:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176924.877098:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786176924.877102:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176924.877107:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786176924.877113:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786176924.877121:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786176924.877130:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.877137:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176924.877141:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.877146:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176924.877149:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786176924.877153:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176924.877156:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176924.877158:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176924.877163:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176924.877169:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.877171:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176924.877175:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176924.877178:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.877180:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176924.877184:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000020:00000040:2.0:1786176924.877188:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176924.877190:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176924.877193:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.877197:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176924.877199:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176924.877203:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176924.877211:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:2.0:1786176924.877216:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176924.877226:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235553664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176924.877245:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176924.877249:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176924.877254:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235553664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176924.877264:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176924.877270:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176924.877277:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786176924.877279:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176924.877282:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176924.877286:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176924.877290:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11599 00000400:00000200:2.0:1786176924.877295:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176924.877299:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176924.877304:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176924.877310:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.877315:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176924.877319:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.877322:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176924.877327:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176924.877330:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786176924.877332:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176924.877336:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786176924.877341:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786176924.877347:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786176924.877351:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.877389:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176924.877398:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176924.877405:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176924.877410:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786176924.877417:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176924.877422:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176924.877426:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176924.877430:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176924.877434:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.877439:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176924.877443:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176924.877448:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.877453:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176924.877457:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000020:00000040:2.0:1786176924.877460:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176924.877465:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176924.877468:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176924.877472:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176924.877475:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176924.877478:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176924.877484:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:2.0:1786176924.877489:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176924.877494:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235553792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176924.877506:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176924.877510:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176924.877515:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235553792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176924.877524:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176924.877529:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11609) 00000800:00000200:1.1:1786176924.877648:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176924.877650:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176924.877897:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176924.877899:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176924.878110:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176924.878112:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786176924.878295:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176924.878297:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00010000:00000001:0.0:1786176925.772460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176925.772470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176925.772481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176925.772490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176925.772501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176925.772506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176925.772561:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176925.772565:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176925.772569:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176926.796506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176926.796517:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176926.796528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176926.796532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176926.796538:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176926.796540:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176926.796625:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176926.796629:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176926.796632:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176927.436497:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176927.436515:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176927.436518:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176927.436527:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176927.436534:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176927.436539:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176927.436543:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176927.436547:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176927.436551:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176927.820448:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176927.820453:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176927.820461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176927.820464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176927.820469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176927.820473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176927.820568:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176927.820576:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176927.820581:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176928.460493:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176928.460514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176928.460518:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176928.460530:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176928.460535:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176928.460540:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176928.460544:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176928.460548:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176928.460551:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176928.780467:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306275584, at 4306275648 00000100:00000010:0.0:1786176928.780482:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ae0. 00010000:00000001:0.0:1786176928.844483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176928.844488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176928.844497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176928.844500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176928.844507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176928.844509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176928.844577:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176928.844583:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176928.844587:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786176929.420620:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1786176929.484515:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176929.484525:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176929.484528:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176929.484535:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176929.484539:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176929.484544:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.484547:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176929.484550:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.484552:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176929.868467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176929.868477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176929.868483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176929.868488:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176929.868492:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176929.868496:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176929.868499:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176929.868501:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176929.868502:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176929.868504:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176929.868506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176929.868507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176929.868573:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176929.868579:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176929.868583:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176929.996752:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176929.996758:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176929.996762:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176929.996772:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bad80 x1872930235553920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.996787:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176929.996790:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176929.996794:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176929.996798:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176929.996805:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176929.996814:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235553920:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176929.996824:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176929.996831:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176929.996837:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176929.996843:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176929.996851:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786176929.996858:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176929.996870:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176929.996878:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176929.996889:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176929.996901:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235553920, portal 25 00000100:00000001:2.0:1786176929.996908:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176929.996928:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487993728 : -110371221557888 : ffff9b9e328bad80) 00000100:00000040:2.0:1786176929.996940:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bad80 x1872930235553920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.996959:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176929.996965:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176929.996969:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176929.996975:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235553920, offset 0 00000400:00000200:2.0:1786176929.996985:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176929.996995:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176929.997003:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176929.997010:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176929.997017:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176929.997023:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176929.997030:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176929.997038:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426394:426394:256:4294967295] 192.168.202.132@tcp LPNI seq info [426356:426356:8:4294967295] 00000400:00000200:2.0:1786176929.997047:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176929.997062:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176929.997071:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176929.997077:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176929.997089:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176929.997094:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176929.997105:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176929.997111:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1786176929.997120:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176929.997123:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176929.997129:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176929.997139:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176929.997146:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176929.997149:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176929.997154:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176929.997161:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bad80 x1872930235553920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997181:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176929.997186:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bad80 x1872930235553920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997198:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.997202:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176929.997208:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176929.997216:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176929.997221:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306280704 00000100:00000010:2.0:1786176929.997229:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290ca80. 00000100:00000001:2.0:1786176929.997238:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786176929.997249:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306280704 00000100:00000010:2.0:1786176929.997256:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cea0. 00000100:00000001:2.0:1786176929.997261:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786176929.997267:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176929.997272:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.997276:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176929.997283:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176929/real 1786176929] req@ffff9b9e328bad80 x1872930235553920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176929.997318:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176929.997321:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176929.997323:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176929.997327:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176929.997330:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176929.997343:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176929.997347:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bad80 x1872930235553920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176929.997391:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bad80 x1872930235553920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997402:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176929.997404:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.997410:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176929.997414:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176929.997427:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bad80 x1872930235553920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997452:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176929.997463:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bad80 x1872930235553920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997481:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176929.997494:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.997499:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176929.997502:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.997507:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176929.997511:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176929.997518:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176929.997527:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176929.997534:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176929.997545:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bad80 x1872930235553920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176929.997559:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235553920:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176929.997565:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.997568:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176929.997576:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bad80 x1872930235553920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997583:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176929.997586:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176929.997588:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786176929.997591:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176929.997593:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176929.997595:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786176929.997598:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176929.997600:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:2.0:1786176929.997603:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176929.997605:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176929.997607:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bad80. 00000100:00000001:2.0:1786176929.997612:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176929.997613:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.997616:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.997624:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176929.997626:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.997628:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176929.997630:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176929.997631:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176929.997636:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328baa00 x1872930235554048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997644:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176929.997645:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176929.997648:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176929.997650:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176929.997652:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176929.997655:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554048:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176929.997659:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176929.997661:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176929.997663:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176929.997664:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176929.997666:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786176929.997668:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176929.997671:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176929.997673:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176929.997676:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176929.997678:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235554048, portal 10 00000100:00000001:2.0:1786176929.997680:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176929.997681:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487992832 : -110371221558784 : ffff9b9e328baa00) 00000100:00000040:2.0:1786176929.997686:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328baa00 x1872930235554048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997690:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176929.997692:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176929.997693:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176929.997695:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235554048, offset 0 00000400:00000200:2.0:1786176929.997698:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176929.997701:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176929.997703:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176929.997705:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176929.997707:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176929.997709:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176929.997711:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176929.997713:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426395:426395:256:4294967295] 192.168.202.132@tcp LPNI seq info [426357:426357:8:4294967295] 00000400:00000200:2.0:1786176929.997716:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176929.997721:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176929.997724:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176929.997726:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176929.997728:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176929.997731:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176929.997734:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176929.997736:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1786176929.997738:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176929.997740:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176929.997742:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176929.997744:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176929.997746:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176929.997747:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176929.997749:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176929.997751:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328baa00 x1872930235554048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997757:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176929.997759:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328baa00 x1872930235554048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997764:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.997765:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176929.997767:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176929.997769:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786176929.997772:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176929.997773:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306276868 00000100:00000001:2.0:1786176929.997775:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306280704 00000100:00000001:2.0:1786176929.997776:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306276868 00000100:00000001:2.0:1786176929.997778:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306280704 00000100:00000001:2.0:1786176929.997779:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176929.997781:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.997782:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176929.997784:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176929/real 1786176929] req@ffff9b9e328baa00 x1872930235554048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176929.997789:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176929.997790:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176929.997791:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176929.997793:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176929.997794:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176929.997796:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176929.997798:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328baa00 x1872930235554048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176929.997803:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328baa00 x1872930235554048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997806:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176929.997807:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.997808:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176929.997809:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176929.997811:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328baa00 x1872930235554048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997815:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176929.997818:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328baa00 x1872930235554048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997822:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176929.997824:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.997825:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176929.997826:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.997827:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176929.997829:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176929.997830:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176929.997832:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176929.997833:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176929.997836:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328baa00 x1872930235554048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176929.997840:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554048:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176929.997843:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176929.997846:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8a80 x1872930235554176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997850:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176929.997851:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176929.997852:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176929.997853:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176929.997854:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176929.997856:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554176:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176929.997858:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176929.997859:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176929.997860:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176929.997861:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176929.997863:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176929.997865:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176929.997867:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176929.997868:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176929.997870:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176929.997872:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235554176, portal 10 00000100:00000001:2.0:1786176929.997873:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176929.997874:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487984768 : -110371221566848 : ffff9b9e328b8a80) 00000100:00000040:2.0:1786176929.997878:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8a80 x1872930235554176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997882:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176929.997884:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176929.997885:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176929.997887:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235554176, offset 0 00000400:00000200:2.0:1786176929.997889:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176929.997891:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176929.997893:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176929.997895:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176929.997897:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176929.997898:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176929.997900:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176929.997902:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426396:426396:256:4294967295] 192.168.202.132@tcp LPNI seq info [426358:426358:8:4294967295] 00000400:00000200:2.0:1786176929.997904:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176929.997909:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176929.997911:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176929.997928:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176929.997932:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176929.997933:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176929.997936:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176929.997938:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1786176929.997941:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176929.997942:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176929.997944:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176929.997946:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176929.997949:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176929.997950:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176929.997951:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176929.997954:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8a80 x1872930235554176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997959:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176929.997961:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8a80 x1872930235554176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.997966:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.997967:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176929.997969:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176929.997971:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786176929.997973:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176929.997974:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306276868 00000100:00000001:2.0:1786176929.997976:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306280704 00000100:00000001:2.0:1786176929.997978:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306276868 00000100:00000001:2.0:1786176929.997980:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306280704 00000100:00000001:2.0:1786176929.997981:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176929.997983:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.997984:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176929.997986:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176929/real 1786176929] req@ffff9b9e328b8a80 x1872930235554176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176929.997991:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176929.997992:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176929.997993:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176929.997994:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176929.997995:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176929.997996:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176929.997998:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8a80 x1872930235554176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176929.998003:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8a80 x1872930235554176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998008:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176929.998008:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998010:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176929.998010:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176929.998013:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8a80 x1872930235554176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998016:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176929.998019:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8a80 x1872930235554176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998023:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176929.998024:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998025:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176929.998026:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998028:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176929.998029:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176929.998031:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176929.998033:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176929.998034:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176929.998037:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8a80 x1872930235554176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176929.998041:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554176:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176929.998044:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176929.998046:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9180 x1872930235554304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998050:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176929.998052:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176929.998053:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176929.998054:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176929.998055:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176929.998057:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554304:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176929.998059:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176929.998060:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176929.998061:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176929.998063:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176929.998065:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786176929.998066:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176929.998068:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176929.998070:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176929.998072:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176929.998074:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235554304, portal 4 00000100:00000001:2.0:1786176929.998075:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176929.998076:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487986560 : -110371221565056 : ffff9b9e328b9180) 00000100:00000040:2.0:1786176929.998080:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9180 x1872930235554304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998084:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176929.998086:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176929.998087:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176929.998089:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235554304, offset 0 00000400:00000200:2.0:1786176929.998091:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176929.998093:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176929.998096:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176929.998098:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176929.998099:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176929.998101:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176929.998103:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176929.998105:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426397:426397:256:4294967295] 192.168.202.132@tcp LPNI seq info [426359:426359:8:4294967295] 00000400:00000200:2.0:1786176929.998107:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176929.998112:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176929.998115:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176929.998117:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176929.998119:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176929.998121:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176929.998124:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176929.998126:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1786176929.998128:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176929.998130:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176929.998132:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176929.998134:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176929.998136:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176929.998137:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176929.998138:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176929.998141:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9180 x1872930235554304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998146:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176929.998147:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9180 x1872930235554304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998156:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.998157:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176929.998159:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176929.998160:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786176929.998162:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176929.998163:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306276868 00000100:00000001:2.0:1786176929.998165:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306280704 00000100:00000001:2.0:1786176929.998167:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306276868 00000100:00000001:2.0:1786176929.998168:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306280704 00000100:00000001:2.0:1786176929.998170:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176929.998171:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.998172:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176929.998174:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176929/real 1786176929] req@ffff9b9e328b9180 x1872930235554304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176929.998178:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176929.998179:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176929.998180:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176929.998182:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176929.998183:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176929.998184:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176929.998186:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9180 x1872930235554304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176929.998191:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9180 x1872930235554304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998195:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176929.998196:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998197:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176929.998197:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176929.998200:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9180 x1872930235554304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998204:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176929.998207:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9180 x1872930235554304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998211:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176929.998212:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998213:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176929.998214:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998215:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176929.998216:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176929.998217:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176929.998219:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176929.998221:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176929.998223:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9180 x1872930235554304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176929.998227:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554304:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176929.998230:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176929.998232:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb800 x1872930235554432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998235:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176929.998236:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176929.998237:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176929.998239:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176929.998240:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176929.998241:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554432:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176929.998243:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176929.998244:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176929.998246:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176929.998247:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176929.998249:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176929.998250:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176929.998252:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176929.998254:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176929.998256:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176929.998257:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235554432, portal 4 00000100:00000001:2.0:1786176929.998258:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176929.998259:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487996416 : -110371221555200 : ffff9b9e328bb800) 00000100:00000040:2.0:1786176929.998263:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb800 x1872930235554432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998267:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176929.998268:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176929.998269:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176929.998271:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235554432, offset 0 00000400:00000200:2.0:1786176929.998273:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176929.998275:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176929.998277:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176929.998279:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176929.998280:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176929.998282:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176929.998283:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176929.998285:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426398:426398:256:4294967295] 192.168.202.132@tcp LPNI seq info [426360:426360:8:4294967295] 00000400:00000200:2.0:1786176929.998288:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176929.998292:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176929.998295:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176929.998296:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176929.998298:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176929.998300:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1786176929.998303:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176929.998304:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1786176929.998307:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176929.998308:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176929.998310:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176929.998313:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176929.998315:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176929.998316:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176929.998319:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176929.998322:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb800 x1872930235554432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998330:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176929.998334:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb800 x1872930235554432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176954 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998341:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.998343:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176929.998346:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176929.998349:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786176929.998351:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176929.998353:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306276870 00000100:00000001:2.0:1786176929.998356:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306280704 00000100:00000001:2.0:1786176929.998378:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306276871 00000100:00000001:2.0:1786176929.998381:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306280704 00000100:00000001:2.0:1786176929.998383:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176929.998385:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.998387:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176929.998391:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176929/real 1786176929] req@ffff9b9e328bb800 x1872930235554432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176929.998400:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176929.998401:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176929.998403:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176929.998407:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176929.998408:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176929.998411:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176929.998415:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb800 x1872930235554432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176929.998423:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb800 x1872930235554432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998430:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176929.998432:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998434:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176929.998435:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176929.998439:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb800 x1872930235554432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998446:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176929.998450:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb800 x1872930235554432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998457:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176929.998459:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998461:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176929.998462:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998464:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176929.998466:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176929.998469:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176929.998472:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176929.998474:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176929.998478:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb800 x1872930235554432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176929.998487:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554432:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176929.998491:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998493:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176929.998497:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328baa00 x1872930235554048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998505:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176929.998506:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176929.998508:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786176929.998511:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176929.998513:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176929.998514:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176929.998517:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176929.998519:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:2.0:1786176929.998521:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176929.998523:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176929.998524:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328baa00. 00000100:00000001:2.0:1786176929.998527:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176929.998529:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998531:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176929.998534:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8a80 x1872930235554176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998541:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176929.998543:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176929.998544:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176929.998546:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176929.998548:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176929.998549:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176929.998551:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176929.998552:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af800. 02000000:00000001:2.0:1786176929.998554:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176929.998555:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176929.998557:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8a80. 00000100:00000001:2.0:1786176929.998559:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176929.998560:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998562:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176929.998565:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9180 x1872930235554304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998572:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176929.998573:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176929.998575:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786176929.998577:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176929.998578:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176929.998580:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176929.998582:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176929.998583:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af400. 02000000:00000001:2.0:1786176929.998585:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176929.998586:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176929.998588:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9180. 00000100:00000001:2.0:1786176929.998590:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176929.998591:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998593:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176929.998596:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb800 x1872930235554432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176954 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176929.998602:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176929.998604:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176929.998605:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176929.998607:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176929.998608:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176929.998609:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176929.998611:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176929.998613:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:2.0:1786176929.998615:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176929.998616:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176929.998617:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb800. 00000100:00000001:2.0:1786176929.998619:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176929.998621:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998622:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176929.998626:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176929.998627:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176929.998629:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176929.998631:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176930.508462:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176930.508471:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176930.508474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176930.508480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176930.508485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176930.508489:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176930.508492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176930.508495:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176930.508498:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176930.892478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176930.892485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176930.892493:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176930.892497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176930.892505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176930.892510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176930.892610:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176930.892613:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176930.892616:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176931.532453:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176931.532461:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176931.532464:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176931.532469:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176931.532474:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176931.532478:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176931.532480:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176931.532484:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176931.532487:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176931.916487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176931.916493:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176931.916498:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176931.916501:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176931.916506:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176931.916510:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176931.916514:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176931.916517:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176931.916522:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176931.916526:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176931.916530:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176931.916531:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176931.916532:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176931.916533:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176931.916534:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176931.916535:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176931.916536:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176931.916538:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176931.916587:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176931.916591:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176931.916593:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176932.556487:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176932.556494:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176932.556495:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176932.556500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176932.556504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176932.556507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176932.556510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176932.556512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176932.556515:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176932.940477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176932.940483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176932.940492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176932.940497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176932.940507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176932.940510:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176932.940558:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176932.940561:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176932.940564:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176933.900465:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306280704, at 4306280768 00000100:00000010:0.0:1786176933.900486:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290ca80. 00010000:00000001:0.0:1786176933.964426:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176933.964432:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176933.964438:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176933.964443:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176933.964452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176933.964457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176933.964537:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176933.964541:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176933.964544:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176934.988465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176934.988471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176934.988481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176934.988485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176934.988490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176934.988493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176934.988548:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176934.988551:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176934.988554:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176935.116489:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176935.116499:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176935.116501:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176935.116504:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176935.116508:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176935.116513:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11609 00000400:00000200:3.0:1786176935.116520:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176935.116525:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176935.116528:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176935.116530:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116533:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176935.116535:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116537:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176935.116540:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176935.116542:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176935.116544:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176935.116546:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176935.116548:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176935.116550:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176935.116552:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116553:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176935.116555:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116556:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176935.116559:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176935.116561:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116563:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176935.116565:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176935.116568:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176935.116570:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.116572:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176935.116573:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176935.116582:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176935.116584:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176935.116588:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176935.116589:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.116591:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176935.116594:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176935.116597:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176935.116604:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176935.116610:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176935.116619:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176935.116631:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176935.116635:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176935.116641:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176935.116666:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176935.116669:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176935.116671:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176935.116672:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176935.116674:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176935.116676:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176935.116678:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11609 00000400:00000200:3.0:1786176935.116681:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176935.116684:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176935.116686:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176935.116689:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116691:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176935.116693:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116695:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176935.116696:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176935.116697:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000100:00000001:2.0:1786176935.116699:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1786176935.116699:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176935.116701:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00000001:2.0:1786176935.116704:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1786176935.116704:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176935.116706:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786176935.116708:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1786176935.116711:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116713:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176935.116717:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786176935.116718:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116722:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176935.116726:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176935.116731:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116736:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.116738:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1786176935.116738:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786176935.116742:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176935.116745:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176935.116746:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176935.116750:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176935.116751:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.116753:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.116756:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:2.0:1786176935.116759:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235554560:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176935.116763:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1786176935.116763:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1786176935.116768:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1786176935.116769:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.116773:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.116773:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.116779:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:3.0:1786176935.116781:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176935.116786:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000010:2.0:1786176935.116787:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:3.0:1786176935.116789:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176935.116793:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.116794:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.116796:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176935.116800:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1786176935.116804:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:3.0:1786176935.116805:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786176935.116811:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 02000000:00000010:3.0:1786176935.116812:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176935.116819:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786176935.116823:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00100000:3.0:1786176935.116824:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1786176935.116831:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235554560, portal 25 00000100:00000001:3.0:1786176935.116832:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176935.116836:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.116837:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176935.116840:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280810880 : -110371428740736 : ffff9b9e26325180) 00000100:00000040:3.0:1786176935.116842:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786176935.116848:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176935.116853:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176935.116858:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786176935.116861:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1786176935.116862:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:2.0:1786176935.116876:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116881:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:2.0:1786176935.116882:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00080000:3.0:1786176935.116884:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1786176935.116888:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235554560, offset 0 00000100:00000001:3.0:1786176935.116888:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176935.116893:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11609 00000400:00000200:2.0:1786176935.116897:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:3.0:1786176935.116898:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176935.116902:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176935.116904:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176935.116908:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116910:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176935.116911:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116913:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786176935.116915:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:3.0:1786176935.116917:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176935.116920:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176935.116921:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176935.116923:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:2.0:1786176935.116925:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000001:3.0:1786176935.116926:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176935.116930:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786176935.116932:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1786176935.116936:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116938:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786176935.116941:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:3.0:1786176935.116944:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.116948:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786176935.116949:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1786176935.116952:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:2.0:1786176935.116956:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00080000:3.0:1786176935.116960:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:2.0:1786176935.116967:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426399:426399:256:4294967295] 192.168.202.132@tcp LPNI seq info [426361:426361:8:4294967295] 00000100:00000001:3.0:1786176935.116968:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176935.116973:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1786176935.116978:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:3.0:1786176935.116979:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176935.116984:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.116986:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176935.116990:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1786176935.116993:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000008:00000001:3.0:1786176935.116994:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.117000:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786176935.117004:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000010:3.0:1786176935.117005:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000800:00000010:2.0:1786176935.117010:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000020:00000040:3.0:1786176935.117011:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176935.117016:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:2.0:1786176935.117021:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786176935.117021:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.117027:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:2.0:1786176935.117028:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 02000000:00000001:3.0:1786176935.117031:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:2.1:1786176935.117035:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786176935.117037:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786176935.117039:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 02000000:00000010:3.0:1786176935.117044:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000800:00000100:2.0:1786176935.117045:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176935.117048:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000100:00000001:3.0:1786176935.117052:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176935.117055:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1786176935.117060:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786176935.117061:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176935.117068:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000100:00000001:3.0:1786176935.117072:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1786176935.117073:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:3.0:1786176935.117077:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.117079:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1786176935.117083:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176935.117087:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176935.117100:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176935.117104:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786176935.117105:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176935.117110:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786176935.117110:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176935.117114:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176935.117117:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176935.117122:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.117123:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786176935.117127:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1786176935.117129:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11609 00000100:00000001:2.0:1786176935.117134:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1786176935.117134:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176935.117141:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:2.0:1786176935.117143:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:3.0:1786176935.117145:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786176935.117148:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306285824 00000100:00000040:3.0:1786176935.117150:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.117153:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:2.0:1786176935.117156:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c1e0. 00000100:00000040:3.0:1786176935.117156:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.117161:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176935.117162:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176935.117163:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:2.0:1786176935.117165:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1786176935.117165:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176935.117169:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176935.117171:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176935.117173:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:2.0:1786176935.117174:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306285824 00000100:00000040:3.0:1786176935.117175:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000010:2.0:1786176935.117181:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c480. 00000100:00000001:3.0:1786176935.117181:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176935.117183:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176935.117185:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1786176935.117187:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786176935.117190:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1786176935.117190:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:2.0:1786176935.117196:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176935.117197:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176935.117200:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1786176935.117201:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176935.117204:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1786176935.117208:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176935/real 1786176935] req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786176935.117208:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176935.117214:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.117219:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176935.117222:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000001:2.0:1786176935.117224:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:3.0:1786176935.117226:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1786176935.117227:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1786176935.117229:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786176935.117230:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000100:00000010:3.0:1786176935.117233:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000400:00000200:2.0:1786176935.117235:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000020:00000040:3.0:1786176935.117237:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000400:00000010:2.0:1786176935.117238:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 02000000:00000001:3.0:1786176935.117242:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786176935.117243:0:2392:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1786176935.117244:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786176935.117248:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176935.117249:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176935.117251:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176935.117254:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176935.117259:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176935.117264:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786176935.117266:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786176935.117271:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176935.117278:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176935.117280:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176935.117282:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176935.117283:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176935.117286:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176935.117286:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176935.117290:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176935.117296:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176935.117299:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1786176935.117303:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786176935.117304:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176935.117307:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176935.117309:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176935.117315:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786176935.117320:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786176935.117321:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11609 00000100:00000040:2.0:1786176935.117326:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1786176935.117332:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176935.117340:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:3.0:1786176935.117341:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786176935.117345:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786176935.117346:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786176935.117347:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176935.117350:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.117354:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:3.0:1786176935.117355:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.117381:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176935.117384:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.117386:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786176935.117387:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1786176935.117388:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176935.117391:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00080000:2.0:1786176935.117393:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1786176935.117394:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176935.117398:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00080000:2.0:1786176935.117401:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000020:00000001:3.0:1786176935.117402:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786176935.117406:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:3.0:1786176935.117407:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176935.117415:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:2.0:1786176935.117416:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176935.117417:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176935.117422:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176935.117427:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176935.117432:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00100000:2.0:1786176935.117437:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235554560:192.168.202.132@tcp:250:kworker.0 00000100:00080000:3.0:1786176935.117437:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176935.117443:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.117447:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1786176935.117447:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786176935.117452:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1786176935.117452:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1786176935.117457:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325180 x1872930235554560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786176935.117458:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.117462:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176935.117465:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1786176935.117470:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000008:00000001:3.0:1786176935.117471:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.117474:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1786176935.117474:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176935.117479:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 00000100:00000010:3.0:1786176935.117479:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 02000000:00000001:2.0:1786176935.117485:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000040:3.0:1786176935.117485:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176935.117489:0:2392:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:3.0:1786176935.117490:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000020:00000040:2.0:1786176935.117492:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176935.117493:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176935.117495:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:3.0:1786176935.117496:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:2.0:1786176935.117498:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:3.0:1786176935.117499:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176935.117501:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176935.117504:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176935.117508:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.117509:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176935.117512:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:3.0:1786176935.117515:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:2.0:1786176935.117516:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325180. 00000100:00000001:2.0:1786176935.117521:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176935.117523:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.117527:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.117536:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176935.117536:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176935.117541:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176935.117542:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.117546:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1786176935.117548:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117549:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176935.117552:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176935.117558:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176935.117560:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176935.117567:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11619) 00000100:00000001:2.0:1786176935.117575:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176935.117577:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.117579:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176935.117580:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176935.117582:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176935.117585:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554688:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176935.117588:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176935.117590:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176935.117591:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.117592:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176935.117594:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176935.117596:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176935.117599:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176935.117600:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176935.117603:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176935.117605:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235554688, portal 10 00000100:00000001:2.0:1786176935.117606:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176935.117607:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818048 : -110371428733568 : ffff9b9e26326d80) 00000100:00000040:2.0:1786176935.117612:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117617:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176935.117619:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176935.117620:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176935.117622:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235554688, offset 0 00000400:00000200:2.0:1786176935.117624:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176935.117627:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176935.117630:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176935.117632:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176935.117634:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176935.117635:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176935.117637:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176935.117640:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426400:426400:256:4294967295] 192.168.202.132@tcp LPNI seq info [426362:426362:8:4294967295] 00000400:00000200:2.0:1786176935.117643:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176935.117647:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176935.117650:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176935.117652:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176935.117655:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176935.117657:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176935.117660:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176935.117662:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1786176935.117665:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176935.117666:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176935.117668:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176935.117670:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176935.117672:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176935.117673:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176935.117675:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176935.117677:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117683:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176935.117684:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117689:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.117690:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176935.117692:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176935.117695:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786176935.117697:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176935.117698:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306281988 00000100:00000001:2.0:1786176935.117700:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306285824 00000100:00000001:2.0:1786176935.117702:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306281988 00000100:00000001:2.0:1786176935.117703:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306285824 00000100:00000001:2.0:1786176935.117705:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176935.117707:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.117708:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176935.117710:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176935/real 1786176935] req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176935.117716:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176935.117716:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176935.117717:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176935.117719:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176935.117720:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176935.117721:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176935.117723:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176935.117728:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117732:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176935.117733:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.117734:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176935.117735:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176935.117737:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117741:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176935.117744:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117748:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176935.117750:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.117751:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176935.117752:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.117753:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176935.117755:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176935.117757:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176935.117759:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176935.117761:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176935.117763:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176935.117770:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554688:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176935.117774:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176935.117777:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117784:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176935.117786:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.117788:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176935.117789:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176935.117791:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176935.117795:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554816:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176935.117798:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176935.117800:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176935.117801:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.117804:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176935.117807:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786176935.117809:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176935.117813:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176935.117816:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176935.117819:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176935.117822:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235554816, portal 10 00000100:00000001:2.0:1786176935.117823:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176935.117824:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280820736 : -110371428730880 : ffff9b9e26327800) 00000100:00000040:2.0:1786176935.117829:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117834:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176935.117836:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176935.117837:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176935.117839:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235554816, offset 0 00000400:00000200:2.0:1786176935.117841:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176935.117844:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176935.117847:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176935.117849:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176935.117851:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176935.117853:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176935.117855:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176935.117857:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426401:426401:256:4294967295] 192.168.202.132@tcp LPNI seq info [426363:426363:8:4294967295] 00000400:00000200:2.0:1786176935.117859:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176935.117879:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176935.117883:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176935.117885:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176935.117888:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176935.117890:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176935.117893:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176935.117895:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1786176935.117898:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176935.117899:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176935.117902:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176935.117904:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176935.117906:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176935.117907:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176935.117909:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176935.117912:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117917:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176935.117919:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117924:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.117925:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176935.117927:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176935.117930:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786176935.117932:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176935.117933:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306281988 00000100:00000001:2.0:1786176935.117935:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306285824 00000100:00000001:2.0:1786176935.117937:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306281988 00000100:00000001:2.0:1786176935.117939:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306285824 00000100:00000001:2.0:1786176935.117942:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176935.117944:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.117946:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176935.117950:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176935/real 1786176935] req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176935.117958:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176935.117959:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176935.117961:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176935.117963:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176935.117965:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176935.117967:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176935.117971:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176935.117981:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.117988:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176935.117990:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.117991:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176935.117993:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176935.117996:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118001:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176935.118004:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118008:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176935.118010:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118012:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176935.118012:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118014:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176935.118015:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176935.118017:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176935.118018:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176935.118020:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176935.118022:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176935.118028:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554816:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176935.118030:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176935.118032:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118037:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176935.118038:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.118039:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176935.118041:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176935.118042:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176935.118044:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554944:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176935.118046:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176935.118047:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176935.118048:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.118050:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176935.118052:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176935.118053:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176935.118056:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176935.118058:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176935.118060:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176935.118063:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235554944, portal 4 00000100:00000001:2.0:1786176935.118064:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176935.118065:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280814464 : -110371428737152 : ffff9b9e26325f80) 00000100:00000040:2.0:1786176935.118069:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118073:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176935.118075:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176935.118076:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176935.118078:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235554944, offset 0 00000400:00000200:2.0:1786176935.118081:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176935.118083:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176935.118086:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176935.118087:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176935.118089:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176935.118091:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176935.118093:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176935.118095:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426402:426402:256:4294967295] 192.168.202.132@tcp LPNI seq info [426364:426364:8:4294967295] 00000400:00000200:2.0:1786176935.118098:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176935.118102:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176935.118105:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176935.118107:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176935.118110:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176935.118112:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176935.118115:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176935.118116:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1786176935.118119:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176935.118121:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176935.118123:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176935.118125:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176935.118127:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176935.118128:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176935.118130:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176935.118133:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118138:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176935.118140:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118144:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.118145:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176935.118147:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176935.118149:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786176935.118151:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176935.118152:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306281988 00000100:00000001:2.0:1786176935.118154:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306285824 00000100:00000001:2.0:1786176935.118155:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306281988 00000100:00000001:2.0:1786176935.118157:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306285824 00000100:00000001:2.0:1786176935.118158:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176935.118159:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.118160:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176935.118162:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176935/real 1786176935] req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176935.118167:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176935.118168:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176935.118169:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176935.118170:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176935.118171:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176935.118173:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176935.118175:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176935.118180:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118185:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176935.118186:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118187:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176935.118188:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176935.118190:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118194:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176935.118197:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118201:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176935.118202:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118203:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176935.118204:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118205:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176935.118206:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176935.118207:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176935.118209:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176935.118210:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176935.118213:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176935.118217:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235554944:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176935.118219:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118221:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176935.118223:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326d80 x1872930235554688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118227:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176935.118228:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176935.118230:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176935.118232:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176935.118233:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176935.118234:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176935.118235:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176935.118237:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:2.0:1786176935.118238:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176935.118239:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176935.118240:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326d80. 00000100:00000001:2.0:1786176935.118242:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176935.118243:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118244:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176935.118246:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327800 x1872930235554816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118250:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176935.118251:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176935.118252:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786176935.118253:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176935.118254:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176935.118255:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176935.118256:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176935.118257:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:2.0:1786176935.118258:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176935.118259:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176935.118259:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327800. 00000100:00000001:2.0:1786176935.118260:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176935.118261:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118262:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176935.118264:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325f80 x1872930235554944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118268:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176935.118268:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176935.118269:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176935.118270:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176935.118271:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176935.118272:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176935.118273:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176935.118274:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:2.0:1786176935.118276:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176935.118276:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176935.118277:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325f80. 00000100:00000001:2.0:1786176935.118279:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176935.118279:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118280:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118283:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176935.118284:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.118285:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176935.118286:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176935.118286:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176935.118288:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118293:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176935.118294:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.118295:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176935.118296:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176935.118297:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176935.118299:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555072:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176935.118301:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176935.118302:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176935.118303:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176935.118304:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176935.118306:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176935.118307:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176935.118309:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176935.118311:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176935.118313:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176935.118314:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235555072, portal 4 00000100:00000001:2.0:1786176935.118316:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176935.118316:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280819840 : -110371428731776 : ffff9b9e26327480) 00000100:00000040:2.0:1786176935.118320:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118328:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176935.118330:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176935.118332:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176935.118333:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235555072, offset 0 00000400:00000200:2.0:1786176935.118335:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176935.118337:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176935.118339:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176935.118341:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176935.118342:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176935.118344:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176935.118345:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176935.118347:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426403:426403:256:4294967295] 192.168.202.132@tcp LPNI seq info [426365:426365:8:4294967295] 00000400:00000200:2.0:1786176935.118350:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176935.118354:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176935.118372:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176935.118375:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176935.118377:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176935.118379:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1786176935.118382:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176935.118384:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:2.0:1786176935.118387:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176935.118388:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176935.118390:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176935.118392:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176935.118394:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176935.118395:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176935.118397:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176935.118399:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118405:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176935.118407:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176960 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118415:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.118417:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176935.118418:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176935.118420:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786176935.118422:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176935.118424:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306281991 00000100:00000001:2.0:1786176935.118426:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306285824 00000100:00000001:2.0:1786176935.118427:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306281991 00000100:00000001:2.0:1786176935.118429:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306285824 00000100:00000001:2.0:1786176935.118430:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176935.118431:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.118432:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176935.118435:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176935/real 1786176935] req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176935.118439:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176935.118440:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176935.118441:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176935.118443:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176935.118444:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176935.118445:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176935.118447:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176935.118453:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118456:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176935.118457:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118458:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176935.118459:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176935.118461:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118467:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176935.118471:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118478:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176935.118481:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118483:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176935.118484:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118486:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176935.118488:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176935.118490:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176935.118493:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176935.118496:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176935.118500:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176935.118507:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555072:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176935.118511:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118513:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176935.118516:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327480 x1872930235555072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176960 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176935.118521:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176935.118523:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176935.118525:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176935.118528:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176935.118529:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176935.118531:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176935.118533:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176935.118534:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:2.0:1786176935.118536:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176935.118538:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176935.118540:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327480. 00000100:00000001:2.0:1786176935.118542:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176935.118543:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118545:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176935.118549:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176935.118550:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176935.118552:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176935.118553:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176936.012468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176936.012472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176936.012478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176936.012482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176936.012486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176936.012488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176936.012538:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176936.012542:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176936.012544:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176937.036452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176937.036458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176937.036464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176937.036467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176937.036473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176937.036474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176937.036552:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176937.036555:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176937.036558:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176937.676454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176937.676462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176937.676465:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176937.676471:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176937.676475:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176937.676479:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176937.676482:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176937.676485:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176937.676489:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176938.060464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176938.060469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176938.060474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176938.060477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176938.060484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176938.060486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176938.060545:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176938.060550:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176938.060553:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176938.700473:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176938.700482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176938.700485:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176938.700491:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176938.700496:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176938.700501:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176938.700504:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176938.700508:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176938.700511:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176939.020447:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306285824, at 4306285888 00000100:00000010:0.0:1786176939.020465:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c1e0. 00010000:00000001:0.0:1786176939.084511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176939.084515:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176939.084519:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176939.084520:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176939.084522:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176939.084523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176939.084566:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176939.084570:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176939.084573:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176939.724507:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176939.724516:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176939.724518:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176939.724526:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176939.724530:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176939.724535:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176939.724538:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176939.724541:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176939.724545:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176940.108503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176940.108516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176940.108529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176940.108536:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176940.108543:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176940.108547:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176940.108554:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176940.108559:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176940.108562:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176940.108564:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176940.108569:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176940.108573:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176940.108641:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176940.108646:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176940.108650:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176940.236446:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176940.236460:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176940.236463:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176940.236468:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176940.236474:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176940.236483:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11614 00000400:00000200:3.0:1786176940.236495:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176940.236502:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176940.236507:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176940.236510:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.236514:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176940.236516:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.236518:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176940.236522:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176940.236524:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176940.236527:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176940.236530:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176940.236532:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176940.236534:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176940.236536:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.236538:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176940.236540:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.236541:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176940.236544:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176940.236547:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176940.236549:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176940.236552:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176940.236555:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176940.236557:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.236559:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176940.236562:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176940.236573:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786176940.236577:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176940.236581:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176940.236582:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.236586:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176940.236589:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176940.236594:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176940.236602:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176940.236609:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176940.236619:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176940.236637:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176940.236642:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176940.236649:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176940.236676:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176940.236680:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176940.236684:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176940.236686:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176940.236688:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176940.236691:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176940.236694:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11614 00000400:00000200:3.0:1786176940.236697:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176940.236707:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176940.236709:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176940.236711:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.236713:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176940.236715:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.236718:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176940.236720:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176940.236721:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176940.236724:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176940.236726:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176940.236728:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176940.236730:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176940.236732:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.236734:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176940.236735:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.236736:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176940.236739:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:2.0:1786176940.236742:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1786176940.236742:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176940.236753:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.236755:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1786176940.236759:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786176940.236760:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1786176940.236767:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176940.236773:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.236776:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:2.0:1786176940.236777:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1786176940.236781:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176940.236792:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.236798:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.236813:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000010:3.0:1786176940.236830:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000100:00000001:2.0:1786176940.236831:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1786176940.236840:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176940.236841:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1786176940.236849:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176940.236850:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176940.236855:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176940.236855:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.236866:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1786176940.236871:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235555200:192.168.202.132@tcp:250:kworker.0 02000000:00000001:3.0:1786176940.236871:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176940.236879:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.236880:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176940.236888:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:3.0:1786176940.236888:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 02000000:00000001:2.0:1786176940.236892:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.236894:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176940.236897:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1786176940.236903:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1786176940.236907:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786176940.236917:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.236921:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176940.236925:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176940.236928:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786176940.236932:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1786176940.236941:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00000001:3.0:1786176940.236944:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176940.236948:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1786176940.236950:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00080000:3.0:1786176940.236952:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176940.236957:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176940.236959:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1786176940.236961:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235555200, portal 25 00000100:00000001:3.0:1786176940.236964:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786176940.236965:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176940.236971:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280816256 : -110371428735360 : ffff9b9e26326680) 00000100:00080000:3.0:1786176940.236971:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11614 00000400:00000200:3.0:1786176940.236981:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786176940.236989:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1786176940.236989:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176940.236995:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176940.237002:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.237007:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176940.237012:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.237019:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176940.237025:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176940.237028:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176940.237032:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176940.237035:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000100:00000001:2.0:1786176940.237037:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1786176940.237039:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176940.237042:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786176940.237045:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786176940.237053:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:2.0:1786176940.237054:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000001:3.0:1786176940.237064:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1786176940.237067:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235555200, offset 0 00000100:00000040:3.0:1786176940.237070:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.237075:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176940.237078:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:2.0:1786176940.237084:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786176940.237085:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176940.237089:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176940.237091:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176940.237094:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176940.237096:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.237101:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:2.0:1786176940.237104:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000008:00100000:3.0:1786176940.237105:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176940.237110:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.237112:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176940.237115:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786176940.237118:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176940.237121:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1786176940.237123:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 02000000:00000001:3.0:1786176940.237123:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.237130:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176940.237133:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176940.237135:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176940.237139:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:3.0:1786176940.237139:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176940.237148:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176940.237154:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00100000:3.0:1786176940.237160:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786176940.237164:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176940.237179:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.237186:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1786176940.237192:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426404:426404:256:4294967295] 192.168.202.132@tcp LPNI seq info [426366:426366:8:4294967295] 00000100:00000001:3.0:1786176940.237192:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176940.237211:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786176940.237212:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786176940.237227:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176940.237232:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1786176940.237241:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00080000:3.0:1786176940.237243:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176940.237249:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176940.237252:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176940.237259:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176940.237265:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11614 00000800:00000200:2.0:1786176940.237268:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000400:00000200:3.0:1786176940.237276:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176940.237281:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000010:2.0:1786176940.237283:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00080000:3.0:1786176940.237285:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176940.237295:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:2.0:1786176940.237296:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176940.237308:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000100:00000001:3.0:1786176940.237308:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176940.237320:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:2.1:1786176940.237321:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176940.237333:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:2.1:1786176940.237333:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000020:00000001:3.0:1786176940.237343:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000100:2.0:1786176940.237344:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786176940.237352:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000800:00000010:2.0:1786176940.237353:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000020:00000001:3.0:1786176940.237383:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176940.237385:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176940.237388:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176940.237390:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176940.237395:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.237396:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176940.237398:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.237401:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176940.237404:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176940.237406:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:2.0:1786176940.237408:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1786176940.237409:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176940.237413:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1786176940.237418:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 02000000:00000001:3.0:1786176940.237419:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176940.237424:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.237426:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:2.0:1786176940.237428:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000008:00100000:3.0:1786176940.237431:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000010:2.0:1786176940.237432:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000008:00000001:3.0:1786176940.237444:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.237447:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786176940.237448:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786176940.237455:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1786176940.237457:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176940.237466:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176940.237475:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786176940.237476:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1786176940.237484:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176940.237485:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176940.237495:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1786176940.237497:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176940.237499:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1786176940.237500:0:2392:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1786176940.237506:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176940.237512:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:2.0:1786176940.237515:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.237516:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176940.237524:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:3.0:1786176940.237526:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176940.237528:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306290945 00000100:00000010:2.0:1786176940.237543:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c600. 00000100:00000001:3.0:1786176940.237543:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176940.237551:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176940.237557:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.237558:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1786176940.237573:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176940.237575:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786176940.237576:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306290945 00000100:00080000:3.0:1786176940.237579:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176940.237585:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176940.237587:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:2.0:1786176940.237588:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c780. 00000100:00000001:3.0:1786176940.237591:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176940.237595:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11614 00000100:00000001:2.0:1786176940.237599:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000400:00000200:3.0:1786176940.237601:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176940.237608:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176940.237612:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786176940.237613:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1786176940.237618:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176940.237620:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.237625:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786176940.237626:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1786176940.237631:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.237634:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176940.237638:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00100000:2.0:1786176940.237642:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176940/real 1786176940] req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786176940.237642:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176940.237648:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176940.237652:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176940.237660:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176940.237667:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000001:2.0:1786176940.237679:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1786176940.237679:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000001:2.0:1786176940.237683:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176940.237689:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000100:00000001:3.0:1786176940.237689:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786176940.237695:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000100:00000040:3.0:1786176940.237696:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176940.237704:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:2.0:1786176940.237705:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176940.237715:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:3.0:1786176940.237715:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176940.237727:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000200:2.0:1786176940.237728:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176940.237740:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176940.237750:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176940.237757:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1786176940.237762:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786176940.237764:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.237774:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176940.237777:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1786176940.237787:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00000001:3.0:1786176940.237787:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.237791:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176940.237792:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.237795:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176940.237798:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1786176940.237798:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176940.237806:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00100000:2.0:1786176940.237807:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786176940.237825:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176940.237828:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176940.237836:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176940.237841:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176940.237845:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.237851:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1786176940.237854:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176940.237858:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176940.237860:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786176940.237870:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176940.237882:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176940.237893:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.237902:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176940.237906:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.237915:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176940.237921:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786176940.237921:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00080000:2.0:1786176940.237928:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1786176940.237928:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176940.237938:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000040:3.0:1786176940.237938:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.237949:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1786176940.237963:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176940.237964:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786176940.237975:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11624) 00000100:00100000:2.0:1786176940.237979:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235555200:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176940.237990:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.237993:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176940.237999:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326680 x1872930235555200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238007:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176940.238010:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176940.238013:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786176940.238017:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176940.238019:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176940.238021:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786176940.238024:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176940.238027:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa000. 02000000:00000001:2.0:1786176940.238031:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176940.238035:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176940.238037:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326680. 00000100:00000001:2.0:1786176940.238039:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176940.238040:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238042:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238049:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176940.238050:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.238052:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176940.238054:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176940.238055:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176940.238059:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238065:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176940.238066:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176940.238068:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176940.238070:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176940.238071:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176940.238074:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555328:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176940.238076:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176940.238078:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176940.238079:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176940.238080:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176940.238083:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786176940.238084:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176940.238088:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176940.238091:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176940.238095:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176940.238098:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235555328, portal 10 00000100:00000001:2.0:1786176940.238100:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176940.238101:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280807296 : -110371428744320 : ffff9b9e26324380) 00000100:00000040:2.0:1786176940.238109:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238116:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176940.238119:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176940.238122:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176940.238124:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235555328, offset 0 00000400:00000200:2.0:1786176940.238129:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176940.238134:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176940.238139:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176940.238141:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176940.238143:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176940.238145:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176940.238147:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176940.238149:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426405:426405:256:4294967295] 192.168.202.132@tcp LPNI seq info [426367:426367:8:4294967295] 00000400:00000200:2.0:1786176940.238152:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176940.238158:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176940.238161:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176940.238164:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176940.238167:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176940.238169:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786176940.238173:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176940.238174:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1786176940.238177:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176940.238179:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176940.238181:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176940.238183:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176940.238186:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176940.238187:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176940.238189:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176940.238192:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238197:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176940.238199:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238204:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.238205:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176940.238208:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176940.238211:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786176940.238213:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176940.238214:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306287108 00000100:00000001:2.0:1786176940.238216:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306290945 00000100:00000001:2.0:1786176940.238218:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306287108 00000100:00000001:2.0:1786176940.238220:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306290945 00000100:00000001:2.0:1786176940.238221:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176940.238223:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.238224:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176940.238227:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176940/real 1786176940] req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176940.238231:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176940.238232:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176940.238233:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176940.238235:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176940.238237:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176940.238239:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176940.238242:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176940.238251:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238257:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176940.238259:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238261:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176940.238263:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176940.238267:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238275:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176940.238279:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238288:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176940.238290:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238291:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176940.238292:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238293:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176940.238295:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176940.238298:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176940.238300:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176940.238302:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176940.238305:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176940.238310:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555328:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176940.238313:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176940.238316:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238320:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176940.238321:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176940.238322:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176940.238324:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176940.238325:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176940.238327:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555456:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176940.238329:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176940.238332:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176940.238333:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176940.238334:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176940.238337:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176940.238338:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176940.238341:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176940.238342:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176940.238344:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176940.238347:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235555456, portal 10 00000100:00000001:2.0:1786176940.238348:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176940.238349:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818944 : -110371428732672 : ffff9b9e26327100) 00000100:00000040:2.0:1786176940.238353:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238377:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176940.238378:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176940.238380:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176940.238382:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235555456, offset 0 00000400:00000200:2.0:1786176940.238385:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176940.238388:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176940.238391:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176940.238393:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176940.238395:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176940.238396:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176940.238398:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176940.238400:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426406:426406:256:4294967295] 192.168.202.132@tcp LPNI seq info [426368:426368:8:4294967295] 00000400:00000200:2.0:1786176940.238403:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176940.238407:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176940.238410:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176940.238412:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176940.238415:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176940.238417:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786176940.238420:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176940.238422:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1786176940.238424:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176940.238426:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176940.238428:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176940.238430:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176940.238432:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176940.238434:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176940.238436:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176940.238438:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238444:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176940.238446:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238450:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.238451:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176940.238453:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176940.238455:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786176940.238457:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176940.238458:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306287109 00000100:00000001:2.0:1786176940.238460:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306290945 00000100:00000001:2.0:1786176940.238461:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306287109 00000100:00000001:2.0:1786176940.238463:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306290945 00000100:00000001:2.0:1786176940.238464:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176940.238465:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.238467:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176940.238469:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176940/real 1786176940] req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176940.238473:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176940.238474:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176940.238475:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176940.238477:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176940.238478:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176940.238479:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176940.238481:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176940.238486:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238491:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176940.238492:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238493:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176940.238494:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176940.238496:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238500:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176940.238502:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238507:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176940.238508:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238509:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176940.238510:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238511:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176940.238512:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176940.238514:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176940.238516:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176940.238517:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176940.238519:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176940.238524:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555456:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176940.238526:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176940.238528:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238533:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176940.238534:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176940.238536:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176940.238537:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176940.238539:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176940.238540:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555584:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176940.238543:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176940.238544:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176940.238545:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176940.238546:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176940.238548:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176940.238549:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176940.238551:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176940.238553:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176940.238555:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176940.238557:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235555584, portal 4 00000100:00000001:2.0:1786176940.238558:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176940.238559:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280819840 : -110371428731776 : ffff9b9e26327480) 00000100:00000040:2.0:1786176940.238565:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238572:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176940.238575:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176940.238577:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176940.238580:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235555584, offset 0 00000400:00000200:2.0:1786176940.238584:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176940.238588:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176940.238592:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176940.238595:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176940.238597:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176940.238600:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176940.238603:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176940.238606:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426407:426407:256:4294967295] 192.168.202.132@tcp LPNI seq info [426369:426369:8:4294967295] 00000400:00000200:2.0:1786176940.238611:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176940.238617:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176940.238621:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176940.238623:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176940.238626:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176940.238628:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786176940.238630:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176940.238632:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1786176940.238636:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176940.238638:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176940.238641:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176940.238645:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176940.238648:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176940.238650:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176940.238653:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176940.238657:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238666:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176940.238670:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238677:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.238679:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176940.238682:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176940.238685:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786176940.238687:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176940.238689:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306287109 00000100:00000001:2.0:1786176940.238691:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306290945 00000100:00000001:2.0:1786176940.238692:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306287109 00000100:00000001:2.0:1786176940.238694:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306290945 00000100:00000001:2.0:1786176940.238695:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176940.238697:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.238698:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176940.238701:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176940/real 1786176940] req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176940.238707:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176940.238708:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176940.238709:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176940.238711:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176940.238712:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176940.238713:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176940.238715:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176940.238720:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238724:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176940.238725:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238726:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176940.238727:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176940.238729:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238733:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176940.238735:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238740:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176940.238742:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238743:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176940.238744:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238745:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176940.238746:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176940.238747:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176940.238749:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176940.238751:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176940.238754:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176940.238759:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555584:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176940.238761:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176940.238763:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238767:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176940.238768:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176940.238770:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176940.238771:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176940.238772:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176940.238774:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555712:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176940.238776:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176940.238777:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176940.238778:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176940.238779:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176940.238782:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786176940.238783:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176940.238785:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176940.238786:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176940.238788:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176940.238790:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235555712, portal 4 00000100:00000001:2.0:1786176940.238792:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176940.238793:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280814464 : -110371428737152 : ffff9b9e26325f80) 00000100:00000040:2.0:1786176940.238796:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238800:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176940.238802:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176940.238803:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176940.238805:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235555712, offset 0 00000400:00000200:2.0:1786176940.238808:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176940.238810:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176940.238813:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176940.238828:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176940.238830:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176940.238832:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176940.238834:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176940.238836:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426408:426408:256:4294967295] 192.168.202.132@tcp LPNI seq info [426370:426370:8:4294967295] 00000400:00000200:2.0:1786176940.238839:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176940.238844:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176940.238847:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176940.238849:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176940.238852:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176940.238854:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1786176940.238857:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176940.238858:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000100:2.0:1786176940.238861:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176940.238862:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176940.238864:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176940.238866:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176940.238869:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176940.238870:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176940.238871:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176940.238874:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238879:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176940.238881:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176965 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238886:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.238887:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176940.238889:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176940.238891:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786176940.238893:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176940.238894:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306287111 00000100:00000001:2.0:1786176940.238896:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306290945 00000100:00000001:2.0:1786176940.238898:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306287111 00000100:00000001:2.0:1786176940.238899:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306290945 00000100:00000001:2.0:1786176940.238900:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176940.238902:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.238903:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176940.238905:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176940/real 1786176940] req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176940.238910:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176940.238911:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176940.238912:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176940.238913:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176940.238914:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176940.238915:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176940.238918:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176940.238923:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238927:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176940.238928:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238929:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176940.238930:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176940.238932:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238936:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176940.238938:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238942:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176940.238944:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238945:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176940.238946:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238947:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176940.238948:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176940.238950:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176940.238952:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176940.238953:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176940.238955:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176940.238960:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555712:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176940.238962:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238964:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176940.238966:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324380 x1872930235555328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.238969:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176940.238971:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176940.238972:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786176940.238973:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176940.238974:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176940.238975:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176940.238977:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176940.238978:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa800. 02000000:00000001:2.0:1786176940.238980:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176940.238981:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176940.238982:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324380. 00000100:00000001:2.0:1786176940.238984:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176940.238985:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.238987:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176940.238990:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327100 x1872930235555456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.239004:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176940.239005:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176940.239007:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176940.239009:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176940.239011:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176940.239012:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176940.239014:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176940.239016:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9000. 02000000:00000001:2.0:1786176940.239018:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176940.239019:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176940.239021:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327100. 00000100:00000001:2.0:1786176940.239023:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176940.239025:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.239027:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176940.239031:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327480 x1872930235555584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.239038:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176940.239039:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176940.239041:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176940.239043:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176940.239044:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176940.239045:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176940.239046:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176940.239048:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9400. 02000000:00000001:2.0:1786176940.239049:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176940.239050:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176940.239051:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327480. 00000100:00000001:2.0:1786176940.239052:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176940.239053:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.239054:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176940.239056:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325f80 x1872930235555712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176965 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176940.239060:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176940.239061:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176940.239062:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786176940.239063:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176940.239064:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176940.239065:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176940.239066:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176940.239067:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:2.0:1786176940.239069:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176940.239069:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176940.239070:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325f80. 00000100:00000001:2.0:1786176940.239072:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176940.239072:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.239073:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.239076:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176940.239077:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.239078:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176940.239080:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176940.748487:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176940.748495:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176940.748497:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176940.748510:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176940.748517:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176940.748523:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.748527:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176940.748533:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176940.748537:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176941.132452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176941.132458:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176941.132464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176941.132465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176941.132469:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176941.132470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176941.132522:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176941.132527:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176941.132531:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176941.772541:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176941.772555:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176941.772557:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176941.772564:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176941.772570:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176941.772575:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176941.772577:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176941.772580:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176941.772582:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176942.156486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176942.156494:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176942.156497:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176942.156502:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176942.156507:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176942.156510:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176942.156513:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176942.156516:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176942.156521:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176942.156524:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176942.156529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176942.156532:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176942.156533:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176942.156535:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176942.156536:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176942.156537:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176942.156538:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176942.156539:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176942.156629:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176942.156637:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176942.156643:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176942.796475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176942.796485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176942.796488:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176942.796495:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176942.796501:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176942.796506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176942.796510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176942.796514:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176942.796517:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176943.180481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176943.180488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176943.180499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176943.180503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176943.180511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176943.180516:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176943.180577:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176943.180582:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176943.180586:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176943.820416:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176943.820425:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176943.820428:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176943.820434:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176943.820438:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176943.820442:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176943.820444:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176943.820447:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176943.820451:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176944.140483:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306290945, at 4306291008 00000100:00000010:0.0:1786176944.140496:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c600. 00010000:00000001:0.0:1786176944.204468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176944.204475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176944.204484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176944.204488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176944.204494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176944.204498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176944.204558:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176944.204562:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176944.204566:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176944.844433:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176944.844441:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176944.844444:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176944.844451:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176944.844455:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176944.844459:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176944.844463:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176944.844465:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176944.844469:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176945.228441:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176945.228448:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176945.228454:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176945.228455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176945.228458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176945.228459:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176945.228498:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176945.228502:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176945.228505:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176945.228594:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176945.228599:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176945.228600:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176945.228611:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176945.228614:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176945.228617:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176945.228619:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176945.228621:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176945.228623:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.356754:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176945.356763:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176945.356778:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176945.356793:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9f80 x1872930235555840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.356818:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176945.356822:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176945.356836:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176945.356842:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176945.356851:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176945.356857:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235555840:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176945.356864:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176945.356871:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176945.356874:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176945.356878:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176945.356888:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786176945.356891:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176945.356902:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176945.356910:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176945.356924:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176945.356932:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235555840, portal 25 00000100:00000001:2.0:1786176945.356936:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176945.356938:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487990144 : -110371221561472 : ffff9b9e328b9f80) 00000100:00000040:2.0:1786176945.356945:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9f80 x1872930235555840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.356974:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176945.356989:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176945.357001:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176945.357012:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235555840, offset 0 00000400:00000200:2.0:1786176945.357030:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176945.357045:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176945.357060:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176945.357071:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176945.357076:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176945.357082:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176945.357084:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176945.357087:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426409:426409:256:4294967295] 192.168.202.132@tcp LPNI seq info [426371:426371:8:4294967295] 00000400:00000200:2.0:1786176945.357094:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176945.357109:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176945.357125:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176945.357131:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176945.357141:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176945.357146:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786176945.357161:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176945.357167:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000100:2.0:1786176945.357180:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176945.357190:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176945.357201:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176945.357220:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176945.357230:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176945.357236:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176945.357249:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176945.357256:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9f80 x1872930235555840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.357278:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176945.357285:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235555840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.357330:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.357337:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176945.357345:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176945.357350:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176945.357396:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306296065 00000100:00000010:2.0:1786176945.357402:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c120. 00000100:00000001:2.0:1786176945.357408:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786176945.357417:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306296065 00000100:00000010:2.0:1786176945.357423:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c540. 00000100:00000001:2.0:1786176945.357429:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786176945.357434:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176945.357442:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.357450:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176945.357455:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176945/real 1786176945] req@ffff9b9e328b9f80 x1872930235555840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176945.357469:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176945.357475:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176945.357478:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176945.357485:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176945.357488:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176945.357493:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176945.357499:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9f80 x1872930235555840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176945.357513:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9f80 x1872930235555840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.357536:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176945.357540:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.357549:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176945.357553:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176945.357565:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9f80 x1872930235555840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.357588:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176945.357598:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9f80 x1872930235555840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.357619:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176945.357623:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.357628:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176945.357630:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.357636:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176945.357640:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176945.357650:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176945.357660:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176945.357669:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176945.357687:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9f80 x1872930235555840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176945.357718:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235555840:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786176945.357733:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.357738:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176945.357744:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9f80 x1872930235555840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.357816:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176945.357821:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176945.357824:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786176945.357829:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176945.357831:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176945.357836:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786176945.357846:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176945.357850:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:2.0:1786176945.357858:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176945.357862:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176945.357869:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9f80. 00000100:00000001:2.0:1786176945.357875:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176945.357880:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.357889:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.357903:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176945.357906:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.357913:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176945.357917:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176945.357920:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176945.357928:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb480 x1872930235555968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.357962:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176945.357965:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176945.357973:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176945.357978:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176945.357988:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176945.358001:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555968:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176945.358018:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176945.358027:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176945.358034:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176945.358037:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176945.358043:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786176945.358048:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176945.358059:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176945.358064:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176945.358070:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176945.358074:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235555968, portal 10 00000100:00000001:2.0:1786176945.358077:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176945.358080:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487995520 : -110371221556096 : ffff9b9e328bb480) 00000100:00000040:2.0:1786176945.358089:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb480 x1872930235555968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358107:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176945.358113:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176945.358122:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176945.358126:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235555968, offset 0 00000400:00000200:2.0:1786176945.358134:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176945.358144:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176945.358153:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176945.358159:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176945.358165:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176945.358173:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176945.358183:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176945.358199:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426410:426410:256:4294967295] 192.168.202.132@tcp LPNI seq info [426372:426372:8:4294967295] 00000400:00000200:2.0:1786176945.358225:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176945.358245:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176945.358251:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176945.358254:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176945.358258:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176945.358261:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786176945.358266:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176945.358268:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000100:2.0:1786176945.358271:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176945.358273:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176945.358275:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176945.358278:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176945.358281:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176945.358282:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176945.358284:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176945.358287:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb480 x1872930235555968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358293:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176945.358295:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb480 x1872930235555968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358300:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.358301:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176945.358303:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176945.358306:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786176945.358309:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176945.358310:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306292228 00000100:00000001:2.0:1786176945.358312:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306296065 00000100:00000001:2.0:1786176945.358314:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306292228 00000100:00000001:2.0:1786176945.358315:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306296065 00000100:00000001:2.0:1786176945.358316:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176945.358318:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.358320:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176945.358322:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176945/real 1786176945] req@ffff9b9e328bb480 x1872930235555968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176945.358327:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176945.358328:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176945.358329:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176945.358330:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176945.358332:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176945.358333:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176945.358335:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb480 x1872930235555968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176945.358341:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb480 x1872930235555968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358345:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176945.358345:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358347:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176945.358348:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176945.358350:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb480 x1872930235555968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358354:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176945.358356:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb480 x1872930235555968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358378:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176945.358379:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358380:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176945.358381:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358383:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176945.358385:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176945.358386:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176945.358389:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176945.358391:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176945.358394:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb480 x1872930235555968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176945.358399:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235555968:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176945.358402:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176945.358404:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb100 x1872930235556096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358408:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176945.358410:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176945.358411:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176945.358412:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176945.358414:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176945.358415:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235556096:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176945.358417:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176945.358419:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176945.358420:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176945.358421:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176945.358424:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176945.358425:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176945.358428:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176945.358429:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176945.358431:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176945.358433:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235556096, portal 10 00000100:00000001:2.0:1786176945.358434:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176945.358435:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487994624 : -110371221556992 : ffff9b9e328bb100) 00000100:00000040:2.0:1786176945.358440:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb100 x1872930235556096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358444:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176945.358446:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176945.358447:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176945.358449:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235556096, offset 0 00000400:00000200:2.0:1786176945.358451:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176945.358454:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176945.358457:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176945.358459:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176945.358461:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176945.358463:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176945.358464:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176945.358466:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426411:426411:256:4294967295] 192.168.202.132@tcp LPNI seq info [426373:426373:8:4294967295] 00000400:00000200:2.0:1786176945.358469:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176945.358473:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176945.358476:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176945.358478:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176945.358480:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176945.358482:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786176945.358485:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176945.358486:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000100:2.0:1786176945.358489:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176945.358490:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176945.358492:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176945.358495:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176945.358497:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176945.358498:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176945.358499:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176945.358502:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb100 x1872930235556096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358507:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176945.358509:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb100 x1872930235556096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358513:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.358514:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176945.358517:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176945.358520:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786176945.358523:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176945.358525:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306292229 00000100:00000001:2.0:1786176945.358528:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306296065 00000100:00000001:2.0:1786176945.358530:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306292229 00000100:00000001:2.0:1786176945.358532:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306296065 00000100:00000001:2.0:1786176945.358534:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176945.358536:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.358538:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176945.358543:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176945/real 1786176945] req@ffff9b9e328bb100 x1872930235556096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176945.358551:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176945.358553:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176945.358555:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176945.358557:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176945.358559:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176945.358561:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176945.358565:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb100 x1872930235556096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176945.358572:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb100 x1872930235556096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358576:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176945.358577:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358579:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176945.358580:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176945.358582:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb100 x1872930235556096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358586:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176945.358589:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb100 x1872930235556096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358593:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176945.358595:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358596:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176945.358597:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358598:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176945.358599:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176945.358601:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176945.358603:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176945.358604:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176945.358606:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb100 x1872930235556096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176945.358612:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235556096:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786176945.358614:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176945.358617:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328ba680 x1872930235556224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358621:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176945.358622:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176945.358624:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176945.358625:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176945.358626:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176945.358628:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235556224:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176945.358630:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176945.358632:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176945.358633:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176945.358634:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176945.358637:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176945.358638:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176945.358641:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176945.358642:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176945.358645:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176945.358647:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235556224, portal 4 00000100:00000001:2.0:1786176945.358648:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176945.358649:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487991936 : -110371221559680 : ffff9b9e328ba680) 00000100:00000040:2.0:1786176945.358654:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328ba680 x1872930235556224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358658:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176945.358659:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176945.358661:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176945.358662:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235556224, offset 0 00000400:00000200:2.0:1786176945.358665:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176945.358667:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176945.358670:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176945.358672:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176945.358673:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176945.358675:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176945.358677:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176945.358679:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426412:426412:256:4294967295] 192.168.202.132@tcp LPNI seq info [426374:426374:8:4294967295] 00000400:00000200:2.0:1786176945.358684:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176945.358690:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176945.358695:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176945.358698:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176945.358702:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176945.358705:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786176945.358709:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176945.358712:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000100:2.0:1786176945.358717:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176945.358719:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176945.358722:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176945.358725:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176945.358729:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176945.358731:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176945.358733:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176945.358736:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328ba680 x1872930235556224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358741:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176945.358743:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328ba680 x1872930235556224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358747:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.358749:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176945.358751:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176945.358753:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786176945.358755:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176945.358757:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306292229 00000100:00000001:2.0:1786176945.358758:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306296065 00000100:00000001:2.0:1786176945.358760:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306292229 00000100:00000001:2.0:1786176945.358761:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306296065 00000100:00000001:2.0:1786176945.358763:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176945.358776:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.358778:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176945.358780:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176945/real 1786176945] req@ffff9b9e328ba680 x1872930235556224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176945.358786:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176945.358787:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176945.358788:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176945.358790:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176945.358791:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176945.358792:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176945.358794:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328ba680 x1872930235556224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176945.358800:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328ba680 x1872930235556224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358804:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176945.358805:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358807:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176945.358808:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176945.358810:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328ba680 x1872930235556224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358814:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176945.358816:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328ba680 x1872930235556224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358820:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176945.358822:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358823:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176945.358823:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358825:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176945.358826:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176945.358828:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176945.358830:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176945.358831:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176945.358834:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328ba680 x1872930235556224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176945.358838:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235556224:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176945.358841:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358842:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176945.358844:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb480 x1872930235555968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358848:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176945.358849:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176945.358851:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786176945.358853:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176945.358854:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176945.358855:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176945.358857:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176945.358858:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:2.0:1786176945.358860:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176945.358861:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176945.358863:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb480. 00000100:00000001:2.0:1786176945.358865:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176945.358866:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358867:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176945.358870:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb100 x1872930235556096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358874:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176945.358875:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176945.358876:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786176945.358877:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176945.358878:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176945.358878:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786176945.358880:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176945.358880:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:2.0:1786176945.358882:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176945.358882:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176945.358883:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb100. 00000100:00000001:2.0:1786176945.358884:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176945.358885:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358886:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176945.358888:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328ba680 x1872930235556224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358891:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176945.358892:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176945.358893:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176945.358894:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176945.358895:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176945.358895:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176945.358897:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176945.358898:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:2.0:1786176945.358899:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176945.358900:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176945.358901:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328ba680. 00000100:00000001:2.0:1786176945.358903:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176945.358904:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358905:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.358908:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176945.358909:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.358910:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176945.358912:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176945.358912:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786176945.358915:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9c00 x1872930235556352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358920:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176945.358921:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176945.358922:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786176945.358924:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176945.358925:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176945.358927:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235556352:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176945.358929:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786176945.358930:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786176945.358932:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176945.358933:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786176945.358935:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176945.358937:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786176945.358939:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786176945.358941:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786176945.358943:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786176945.358945:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235556352, portal 4 00000100:00000001:2.0:1786176945.358946:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786176945.358947:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487989248 : -110371221562368 : ffff9b9e328b9c00) 00000100:00000040:2.0:1786176945.358952:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9c00 x1872930235556352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.358961:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786176945.358962:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786176945.358964:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786176945.358965:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235556352, offset 0 00000400:00000200:2.0:1786176945.358969:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786176945.358973:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176945.358977:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786176945.358980:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786176945.358982:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786176945.358985:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786176945.358988:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786176945.358991:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426413:426413:256:4294967295] 192.168.202.132@tcp LPNI seq info [426375:426375:8:4294967295] 00000400:00000200:2.0:1786176945.358995:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176945.359002:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786176945.359007:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176945.359011:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786176945.359015:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786176945.359018:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1786176945.359022:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786176945.359024:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 23 secs later) 00000800:00000100:2.0:1786176945.359026:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786176945.359028:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786176945.359030:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786176945.359032:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786176945.359035:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786176945.359036:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786176945.359037:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786176945.359040:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9c00 x1872930235556352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.359045:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176945.359047:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9c00 x1872930235556352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176970 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.359056:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.359057:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786176945.359059:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176945.359062:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786176945.359064:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786176945.359065:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306292231 00000100:00000001:2.0:1786176945.359067:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306296065 00000100:00000001:2.0:1786176945.359069:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306292231 00000100:00000001:2.0:1786176945.359070:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306296065 00000100:00000001:2.0:1786176945.359072:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786176945.359073:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.359074:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786176945.359076:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786176945/real 1786176945] req@ffff9b9e328b9c00 x1872930235556352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786176945.359081:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786176945.359082:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786176945.359083:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786176945.359085:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786176945.359086:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786176945.359087:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786176945.359089:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9c00 x1872930235556352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786176945.359095:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9c00 x1872930235556352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.359098:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786176945.359099:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.359100:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176945.359101:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786176945.359103:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9c00 x1872930235556352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.359108:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786176945.359111:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9c00 x1872930235556352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.359115:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786176945.359116:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.359117:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786176945.359118:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.359119:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786176945.359120:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786176945.359122:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786176945.359124:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786176945.359125:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786176945.359127:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9c00 x1872930235556352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786176945.359132:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235556352:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786176945.359134:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.359135:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786176945.359137:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9c00 x1872930235556352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786176970 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176945.359141:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786176945.359142:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786176945.359143:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786176945.359145:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786176945.359146:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786176945.359147:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786176945.359148:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786176945.359149:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:2.0:1786176945.359151:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786176945.359152:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786176945.359153:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9c00. 00000100:00000001:2.0:1786176945.359155:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786176945.359156:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.359157:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.359160:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176945.359161:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.359162:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786176945.359163:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176945.868485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176945.868493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176945.868496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176945.868502:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176945.868505:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176945.868508:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.868510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176945.868512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176945.868514:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176946.252419:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176946.252426:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176946.252427:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176946.252432:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786176946.252435:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176946.252439:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176946.252441:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176946.252443:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176946.252446:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176946.252461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176946.252466:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176946.252471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176946.252472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176946.252476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176946.252477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176946.252521:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176946.252525:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176946.252528:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176946.892412:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176946.892421:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176946.892424:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176946.892430:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176946.892435:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176946.892439:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176946.892441:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176946.892444:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176946.892447:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176947.276427:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786176947.276433:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786176947.276436:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786176947.276437:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176947.276440:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176947.276443:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176947.276445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176947.276447:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786176947.276449:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176947.276451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176947.276455:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176947.276460:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176947.276463:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176947.276466:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176947.276470:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176947.276503:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176947.276507:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176947.276512:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176947.916454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176947.916460:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176947.916462:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176947.916468:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176947.916471:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176947.916475:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176947.916477:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176947.916480:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176947.916483:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176948.300459:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786176948.300463:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786176948.300466:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786176948.300467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176948.300470:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176948.300475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1786176948.300475:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786176948.300478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176948.300480:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786176948.300482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786176948.300484:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786176948.300485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176948.300488:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176948.300490:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176948.300493:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176948.300543:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176948.300546:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176948.300549:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176948.940457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176948.940468:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176948.940471:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176948.940486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176948.940491:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176948.940496:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176948.940499:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176948.940503:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176948.940506:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176949.260441:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306296065, at 4306296128 00000100:00000010:0.0:1786176949.260456:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c120. 00010000:00000001:0.0:1786176949.324439:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176949.324445:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176949.324452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176949.324455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176949.324460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176949.324463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176949.324519:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176949.324523:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176949.324526:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176949.964420:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176949.964430:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176949.964433:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176949.964440:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176949.964446:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176949.964450:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176949.964453:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176949.964456:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176949.964459:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176950.348466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176950.348472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176950.348477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176950.348481:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176950.348483:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176950.348487:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176950.348493:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176950.348495:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176950.348498:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176950.348502:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176950.348504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176950.348506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176950.348564:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176950.348569:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176950.348573:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786176950.476514:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176950.476525:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786176950.476528:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176950.476531:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176950.476536:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176950.476544:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11624 00000400:00000200:2.0:1786176950.476554:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176950.476561:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176950.476565:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176950.476569:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.476572:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176950.476575:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.476577:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176950.476582:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176950.476584:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786176950.476587:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176950.476591:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786176950.476594:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786176950.476597:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786176950.476600:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.476602:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176950.476604:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.476606:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176950.476609:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786176950.476612:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176950.476615:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176950.476618:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176950.476623:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176950.476626:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.476629:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176950.476631:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176950.476641:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000020:00000040:2.0:1786176950.476645:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786176950.476650:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176950.476651:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.476654:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176950.476657:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176950.476662:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176950.476669:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786176950.476675:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176950.476686:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235556480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176950.476697:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176950.476702:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176950.476707:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235556480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176950.476747:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176950.476751:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176950.476755:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786176950.476756:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176950.476758:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176950.476762:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176950.476764:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11624 00000400:00000200:2.0:1786176950.476768:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176950.476771:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176950.476774:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176950.476776:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.476778:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176950.476781:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.476783:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176950.476785:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176950.476786:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786176950.476788:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176950.476791:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786176950.476793:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786176950.476795:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786176950.476799:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.476802:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176950.476805:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.476808:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176950.476810:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786176950.476816:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176950.476822:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176950.476827:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176950.476834:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176950.476841:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.476846:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176950.476852:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176950.476860:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.476865:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176950.476872:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 00000020:00000040:2.0:1786176950.476879:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176950.476885:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176950.476889:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.476894:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176950.476896:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176950.476899:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176950.476905:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:2.0:1786176950.476910:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176950.476918:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235556608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176950.476926:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176950.476931:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176950.476937:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235556608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176950.476953:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176950.476957:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176950.476963:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786176950.476966:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176950.476970:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176950.476975:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176950.476979:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11624 00000400:00000200:2.0:1786176950.476988:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176950.476992:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176950.476994:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176950.476999:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477002:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176950.477006:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477011:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176950.477013:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176950.477015:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786176950.477018:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176950.477021:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786176950.477025:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786176950.477029:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786176950.477035:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477039:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176950.477045:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477050:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176950.477052:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786176950.477057:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477062:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176950.477064:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176950.477068:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176950.477073:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.477075:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176950.477078:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176950.477080:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.477082:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176950.477086:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 00000020:00000040:2.0:1786176950.477091:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176950.477094:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176950.477096:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.477099:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176950.477100:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176950.477103:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176950.477117:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000001:2.0:1786176950.477121:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176950.477127:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235556736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176950.477139:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176950.477141:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176950.477144:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235556736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176950.477149:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176950.477151:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:2.0:1786176950.477153:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1786176950.477159:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00080000:2.0:1786176950.477166:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786176950.477171:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176950.477175:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176950.477181:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176950.477186:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11624 00000400:00000200:2.0:1786176950.477195:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176950.477202:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176950.477206:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176950.477214:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477217:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176950.477222:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477224:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176950.477227:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176950.477230:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786176950.477232:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176950.477235:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786176950.477238:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786176950.477242:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786176950.477247:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477250:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176950.477252:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477256:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176950.477261:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786176950.477267:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477271:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176950.477275:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176950.477279:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176950.477284:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.477286:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176950.477289:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176950.477291:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.477296:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176950.477300:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000020:00000040:2.0:1786176950.477304:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176950.477309:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176950.477310:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.477313:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176950.477315:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176950.477318:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176950.477322:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:2.0:1786176950.477327:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176950.477332:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235556864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176950.477341:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176950.477345:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176950.477352:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235556864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176950.477395:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176950.477402:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176950.477408:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786176950.477412:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176950.477416:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176950.477423:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176950.477430:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11624 00000400:00000200:2.0:1786176950.477440:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176950.477447:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176950.477452:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176950.477458:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477462:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176950.477466:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477470:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176950.477475:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176950.477476:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786176950.477479:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176950.477483:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786176950.477487:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786176950.477491:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786176950.477498:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477502:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176950.477507:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477510:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176950.477513:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786176950.477520:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176950.477526:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176950.477530:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176950.477535:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176950.477542:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.477548:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176950.477551:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176950.477556:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.477560:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176950.477567:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000020:00000040:2.0:1786176950.477571:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176950.477576:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176950.477578:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176950.477583:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176950.477585:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176950.477589:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176950.477594:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9000. 00000100:00000001:2.0:1786176950.477599:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176950.477608:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235556992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176950.477621:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176950.477625:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176950.477631:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235556992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176950.477645:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176950.477651:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11635) 00000800:00000200:1.1:1786176950.477780:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176950.477782:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176950.477965:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176950.477968:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176950.478149:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176950.478151:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786176950.478328:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176950.478330:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1786176950.988441:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176950.988449:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176950.988451:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176950.988455:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176950.988459:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176950.988462:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176950.988464:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176950.988466:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176950.988468:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176951.372465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176951.372469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176951.372474:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176951.372476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176951.372479:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176951.372481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176951.372539:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176951.372543:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176951.372546:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176952.012483:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176952.012491:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176952.012493:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176952.012497:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176952.012501:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176952.012505:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176952.012507:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176952.012509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176952.012512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176952.396468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176952.396476:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176952.396481:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176952.396485:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176952.396489:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176952.396492:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176952.396496:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176952.396500:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176952.396504:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176952.396507:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176952.396510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176952.396514:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176952.396517:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176952.396521:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176952.396525:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176952.396527:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176952.396529:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176952.396531:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176952.396586:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176952.396589:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176952.396592:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176953.036857:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176953.036870:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176953.036874:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176953.036884:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176953.036890:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176953.036895:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176953.036899:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176953.036903:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176953.036907:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176953.420477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176953.420483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176953.420491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176953.420495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176953.420501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176953.420505:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176953.420573:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176953.420578:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176953.420583:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1786176954.380460:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306301184, at 4306301248 00000100:00000010:0.0:1786176954.380477:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62240. 00010000:00000001:0.0:1786176954.444461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176954.444467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176954.444477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176954.444480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176954.444485:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176954.444488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176954.444576:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176954.444580:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176954.444584:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1786176955.468489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176955.468495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176955.468505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176955.468509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176955.468516:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176955.468521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176955.468580:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176955.468583:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176955.468586:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176955.596489:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176955.596497:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176955.596499:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176955.596503:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176955.596508:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176955.596515:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11630 00000400:00000200:3.0:1786176955.596523:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176955.596528:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176955.596530:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176955.596533:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596536:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176955.596538:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596540:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176955.596544:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176955.596545:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176955.596548:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176955.596551:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176955.596553:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176955.596554:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176955.596556:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596558:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176955.596559:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596561:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176955.596563:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176955.596566:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596568:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176955.596570:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176955.596573:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176955.596575:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.596577:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176955.596579:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176955.596587:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176955.596589:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176955.596592:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176955.596594:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.596596:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176955.596599:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176955.596603:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176955.596610:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176955.596615:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176955.596622:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235557120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176955.596630:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176955.596634:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176955.596638:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235557120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176955.596659:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176955.596662:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176955.596664:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176955.596676:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176955.596678:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176955.596681:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176955.596683:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11630 00000400:00000200:3.0:1786176955.596686:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176955.596689:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176955.596691:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176955.596694:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596697:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176955.596699:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596701:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176955.596702:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176955.596704:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176955.596705:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176955.596710:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176955.596712:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176955.596715:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176955.596720:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596722:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176955.596726:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596728:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176955.596732:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176955.596736:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596738:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176955.596739:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176955.596743:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176955.596747:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.596751:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176955.596756:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176955.596760:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.596764:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176955.596771:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176955.596776:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176955.596781:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176955.596785:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.596788:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176955.596792:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176955.596797:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176955.596804:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176955.596811:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176955.596823:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235557248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176955.596837:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176955.596841:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176955.596849:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235557248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176955.596864:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176955.596870:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176955.596874:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176955.596877:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176955.596881:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176955.596886:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176955.596891:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11630 00000400:00000200:3.0:1786176955.596898:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176955.596906:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176955.596911:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176955.596915:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596919:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176955.596925:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596927:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176955.596930:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176955.596932:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176955.596935:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176955.596937:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176955.596939:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176955.596943:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176955.596946:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596949:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176955.596953:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596956:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176955.596958:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176955.596960:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176955.596965:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176955.596969:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176955.596972:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176955.596973:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.596977:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176955.596979:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176955.596983:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.596985:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176955.596989:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176955.596992:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176955.596995:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176955.596997:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.597000:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176955.597002:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176955.597006:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176955.597010:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176955.597015:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176955.597022:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235557376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176955.597035:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176955.597037:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176955.597043:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235557376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176955.597056:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176955.597059:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176955.597063:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176955.597067:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176955.597069:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176955.597072:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176955.597077:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11630 00000400:00000200:3.0:1786176955.597081:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176955.597086:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176955.597089:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176955.597093:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.597097:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176955.597101:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.597104:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:1.1:1786176955.597105:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:3.0:1786176955.597107:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176955.597109:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176955.597112:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176955.597114:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000800:00000200:1.1:1786176955.597114:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000020:00000001:3.0:1786176955.597118:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176955.597121:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176955.597128:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.597131:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176955.597135:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.597138:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176955.597140:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176955.597145:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176955.597148:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176955.597151:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176955.597155:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176955.597158:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.597159:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176955.597161:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176955.597165:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.597167:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176955.597173:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176955.597176:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176955.597178:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176955.597181:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.597182:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176955.597185:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176955.597187:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176955.597192:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176955.597196:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176955.597204:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235557504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176955.597214:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176955.597217:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176955.597224:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235557504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176955.597239:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176955.597245:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176955.597249:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176955.597250:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176955.597251:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176955.597254:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176955.597256:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11630 00000400:00000200:3.0:1786176955.597261:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176955.597267:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176955.597271:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176955.597274:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.597278:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176955.597279:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.597281:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176955.597283:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176955.597286:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176955.597288:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176955.597289:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176955.597291:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176955.597293:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176955.597299:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.597300:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176955.597301:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176955.597303:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176955.597305:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176955.597310:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176955.597312:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176955.597318:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176955.597320:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176955.597324:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.597327:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176955.597328:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176955.597334:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.597336:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176955.597347:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176955.597351:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176955.597383:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176955.597385:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176955.597387:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176955.597389:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176955.597392:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176955.597398:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176955.597402:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176955.597410:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235557632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176955.597420:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176955.597423:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176955.597429:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235557632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176955.597438:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176955.597444:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11640) 00000800:00000200:1.1:1786176955.597733:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176955.597736:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176955.598078:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176955.598081:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176955.598420:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176955.598424:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786176955.598767:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176955.598770:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1786176956.492425:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176956.492430:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176956.492436:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176956.492440:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176956.492444:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176956.492446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176956.492501:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176956.492505:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176956.492508:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176957.133002:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176957.133011:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176957.133014:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176957.133020:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176957.133024:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176957.133028:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176957.133031:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176957.133034:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176957.133037:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176957.516461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176957.516465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176957.516472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176957.516475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176957.516482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176957.516485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176957.516545:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176957.516550:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176957.516554:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176958.157588:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176958.157594:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176958.157596:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176958.157601:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176958.157604:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176958.157608:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176958.157610:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176958.157612:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176958.157615:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176958.540462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176958.540469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176958.540477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176958.540482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176958.540491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176958.540495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176958.540557:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176958.540562:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176958.540565:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176959.180484:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176959.180497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176959.180500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176959.180507:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176959.180513:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176959.180519:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176959.180523:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176959.180528:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176959.180533:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176959.500467:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306306304, at 4306306368 00000100:00000010:0.0:1786176959.500488:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ae0. 00010000:00000001:0.0:1786176959.564515:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176959.564522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176959.564531:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176959.564537:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176959.564542:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176959.564548:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176959.564554:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176959.564559:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176959.564563:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176959.564567:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176959.564573:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176959.564577:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176959.564660:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176959.564666:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176959.564670:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176960.204492:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176960.204503:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176960.204507:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176960.204514:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176960.204520:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176960.204525:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176960.204529:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176960.204533:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176960.204537:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176960.588476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176960.588481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176960.588491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176960.588494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176960.588498:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176960.588501:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176960.588566:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176960.588574:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176960.588580:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176960.716563:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176960.716592:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176960.716595:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176960.716599:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176960.716604:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176960.716611:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11635 00000400:00000200:0.0:1786176960.716635:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176960.716643:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176960.716646:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176960.716649:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.716652:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176960.716654:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.716672:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176960.716676:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176960.716678:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176960.716681:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176960.716684:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176960.716686:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176960.716688:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176960.716690:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.716691:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176960.716692:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.716694:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176960.716696:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176960.716698:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176960.716700:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176960.716702:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176960.716706:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176960.716708:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.716710:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176960.716712:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176960.716723:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786176960.716726:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176960.716731:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176960.716732:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.716736:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176960.716738:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176960.716743:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176960.716753:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176960.716760:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176960.716770:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235557760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176960.716779:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176960.716783:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176960.716787:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235557760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176960.716812:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176960.716815:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176960.716818:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176960.716819:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176960.716820:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176960.716823:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176960.716825:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11635 00000400:00000200:0.0:1786176960.716827:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176960.716829:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176960.716831:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176960.716833:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.716835:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176960.716836:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.716838:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176960.716839:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176960.716840:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176960.716841:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176960.716843:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176960.716846:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176960.716848:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176960.716850:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.716852:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176960.716854:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.716859:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176960.716861:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176960.716866:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176960.716869:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176960.716871:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176960.716877:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176960.716882:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.716886:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176960.716891:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176960.716894:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.716896:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176960.716899:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786176960.716903:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176960.716906:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176960.716907:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.716910:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176960.716912:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176960.716915:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176960.716922:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176960.716926:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176960.716933:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235557888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176960.716946:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176960.716948:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176960.716953:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235557888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176960.716969:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176960.716972:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176960.716989:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176960.716992:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176960.716994:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176960.716996:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176960.716999:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11635 00000400:00000200:0.0:1786176960.717004:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176960.717009:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176960.717013:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176960.717017:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717019:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176960.717023:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717027:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176960.717031:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176960.717032:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176960.717036:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176960.717038:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176960.717042:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176960.717044:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176960.717050:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717052:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176960.717055:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717057:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176960.717059:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176960.717064:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717067:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176960.717068:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176960.717071:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176960.717074:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.717076:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176960.717078:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176960.717080:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.717082:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176960.717105:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786176960.717109:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176960.717112:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176960.717115:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.717120:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176960.717123:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176960.717128:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176960.717136:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176960.717141:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176960.717150:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235558016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176960.717188:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176960.717193:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176960.717202:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235558016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1786176960.717217:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786176960.717223:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176960.717223:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00080000:0.0:1786176960.717231:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176960.717238:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176960.717241:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176960.717243:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176960.717250:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176960.717257:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11635 00000400:00000200:0.0:1786176960.717264:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176960.717270:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176960.717275:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176960.717283:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717288:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176960.717293:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717297:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176960.717302:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176960.717303:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176960.717306:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176960.717309:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176960.717314:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176960.717318:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176960.717324:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717326:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176960.717329:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717335:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176960.717339:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176960.717345:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717350:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176960.717353:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176960.717355:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176960.717357:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.717358:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176960.717361:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176960.717362:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.717379:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176960.717382:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786176960.717384:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176960.717389:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176960.717390:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.717391:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176960.717392:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176960.717394:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176960.717398:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176960.717401:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176960.717407:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235558144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176960.717470:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176960.717492:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176960.717500:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235558144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176960.717518:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176960.717524:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176960.717533:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176960.717536:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176960.717540:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176960.717547:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176960.717553:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11635 00000400:00000200:0.0:1786176960.717561:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176960.717567:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176960.717572:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176960.717579:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717584:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176960.717588:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717592:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176960.717595:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176960.717597:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176960.717601:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176960.717604:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176960.717609:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176960.717613:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176960.717636:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717642:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176960.717647:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717668:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176960.717672:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176960.717678:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176960.717682:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176960.717686:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176960.717691:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176960.717697:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.717701:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176960.717705:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176960.717710:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.717716:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176960.717724:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786176960.717729:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176960.717736:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176960.717740:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176960.717744:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176960.717747:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176960.717751:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176960.717758:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176960.717765:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176960.717775:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235558272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176960.717792:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176960.717796:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176960.717804:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235558272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176960.717822:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176960.717828:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11645) 00000800:00000200:1.1:1786176960.717923:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176960.717924:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176960.718161:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176960.718163:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176960.718431:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176960.718433:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786176960.718715:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176960.718717:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1786176961.228437:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176961.228446:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176961.228449:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176961.228457:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176961.228461:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176961.228466:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176961.228469:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176961.228474:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176961.228477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176961.612499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176961.612507:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176961.612510:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176961.612515:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176961.612527:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176961.612532:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176961.612536:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176961.612540:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176961.612545:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176961.612548:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176961.612555:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176961.612565:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176961.612573:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176961.612577:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176961.612580:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176961.612582:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176961.612583:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176961.612585:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176961.612671:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176961.612675:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176961.612677:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176962.252533:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176962.252543:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176962.252545:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176962.252550:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176962.252555:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176962.252559:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176962.252562:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176962.252564:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176962.252567:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176962.636464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176962.636480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176962.636495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176962.636497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176962.636502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176962.636503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176962.636555:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176962.636559:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176962.636562:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176963.276475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176963.276485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176963.276488:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176963.276495:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176963.276500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176963.276504:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176963.276507:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176963.276510:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176963.276514:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176963.660465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176963.660471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176963.660490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176963.660493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176963.660497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176963.660498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176963.660555:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176963.660560:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176963.660563:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176964.300475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176964.300482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176964.300484:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176964.300489:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176964.300493:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176964.300496:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176964.300499:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176964.300507:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176964.300510:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176964.620465:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306311424, at 4306311488 00000100:00000010:0.0:1786176964.620482:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62d20. 00010000:00000001:0.0:1786176964.684434:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176964.684438:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176964.684444:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176964.684446:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176964.684450:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176964.684453:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176964.684504:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176964.684509:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176964.684513:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786176965.324492:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176965.324501:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176965.324504:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176965.324510:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176965.324515:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176965.324519:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176965.324522:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176965.324525:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176965.324529:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176965.708492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176965.708500:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176965.708528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176965.708533:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176965.708541:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176965.708544:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176965.708621:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176965.708627:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176965.708631:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176965.836437:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176965.836448:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176965.836451:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176965.836455:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176965.836460:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176965.836469:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11640 00000400:00000200:0.0:1786176965.836478:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176965.836484:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176965.836487:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176965.836490:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836493:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176965.836495:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836497:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176965.836501:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176965.836503:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176965.836506:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176965.836508:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176965.836510:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176965.836512:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176965.836514:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836515:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176965.836517:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836518:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176965.836521:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176965.836524:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836525:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176965.836528:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176965.836532:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176965.836534:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.836536:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176965.836538:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176965.836549:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176965.836553:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176965.836558:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176965.836559:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.836562:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176965.836565:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176965.836616:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176965.836627:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786176965.836635:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176965.836652:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235558400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176965.836661:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176965.836666:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176965.836670:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235558400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176965.836695:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176965.836697:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176965.836700:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176965.836702:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176965.836703:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176965.836706:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176965.836708:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11640 00000400:00000200:0.0:1786176965.836711:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176965.836713:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176965.836715:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176965.836717:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836719:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176965.836721:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836722:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176965.836724:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176965.836725:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176965.836727:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176965.836729:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176965.836731:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176965.836732:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176965.836734:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836735:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176965.836736:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836738:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176965.836740:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176965.836742:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836743:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176965.836745:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176965.836751:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176965.836753:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.836755:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176965.836759:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176965.836762:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.836763:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176965.836766:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176965.836768:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176965.836771:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176965.836773:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.836776:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176965.836778:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176965.836780:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176965.836785:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786176965.836790:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176965.836795:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235558528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176965.836802:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176965.836804:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176965.836810:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235558528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176965.836827:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176965.836833:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176965.836839:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176965.836840:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176965.836841:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176965.836843:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176965.836846:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11640 00000400:00000200:0.0:1786176965.836853:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176965.836860:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176965.836865:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176965.836869:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836872:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176965.836875:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836878:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176965.836880:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176965.836883:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176965.836886:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176965.836888:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176965.836890:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176965.836893:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176965.836896:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836898:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176965.836902:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836908:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176965.836917:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176965.836924:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176965.836927:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176965.836932:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176965.836935:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176965.836942:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.836944:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176965.836948:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176965.836951:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.836952:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176965.836956:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176965.836959:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176965.836965:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176965.836966:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.836968:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176965.836971:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176965.836973:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176965.836978:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786176965.836982:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176965.836991:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235558656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176965.836999:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176965.837002:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176965.837009:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235558656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176965.837022:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176965.837027:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176965.837031:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176965.837033:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176965.837036:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176965.837042:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176965.837045:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11640 00000400:00000200:0.0:1786176965.837048:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176965.837051:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176965.837058:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176965.837061:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.837062:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176965.837064:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.837067:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176965.837072:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176965.837074:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176965.837077:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176965.837079:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176965.837085:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176965.837089:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176965.837092:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.837095:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176965.837098:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.837103:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176965.837109:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176965.837114:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176965.837121:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176965.837125:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176965.837129:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176965.837136:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.837142:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176965.837147:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176965.837157:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.837162:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176965.837169:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176965.837175:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176965.837181:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176965.837183:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.837185:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176965.837187:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176965.837189:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176965.837197:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176965.837203:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176965.837212:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235558784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176965.837233:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176965.837239:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176965.837248:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235558784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176965.837261:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176965.837264:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176965.837267:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176965.837269:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176965.837274:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176965.837278:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176965.837286:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11640 00000400:00000200:0.0:1786176965.837325:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176965.837333:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176965.837337:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176965.837343:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.837345:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176965.837348:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.837354:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176965.837376:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176965.837378:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176965.837379:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176965.837380:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176965.837382:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176965.837384:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176965.837387:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.837388:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176965.837390:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176965.837391:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176965.837393:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176965.837395:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176965.837397:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176965.837398:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176965.837401:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176965.837403:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.837404:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176965.837406:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176965.837408:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.837409:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176965.837412:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176965.837417:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176965.837422:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176965.837425:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176965.837428:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176965.837429:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176965.837436:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176965.837441:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176965.837447:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176965.837449:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00100000:0.0:1786176965.837454:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235558912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1786176965.837457:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000001:0.0:1786176965.837473:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176965.837480:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176965.837491:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235558912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176965.837505:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176965.837514:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11650) 00000800:00000200:1.1:1786176965.837984:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176965.837987:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176965.838317:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176965.838320:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176965.838685:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176965.838688:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786176965.839003:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176965.839006:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1786176966.348506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176966.348515:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176966.348517:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176966.348523:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176966.348527:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176966.348533:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176966.348536:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176966.348539:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176966.348542:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176966.732488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176966.732499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176966.732518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176966.732522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176966.732526:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176966.732528:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176966.732595:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176966.732599:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176966.732602:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786176967.308743:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1786176967.372482:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786176967.372491:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786176967.372494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176967.372499:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786176967.372502:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786176967.372506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176967.372508:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786176967.372512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786176967.372515:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176967.756442:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176967.756451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176967.756459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176967.756463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176967.756472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176967.756476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176967.756530:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176967.756533:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176967.756536:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176968.396531:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176968.396538:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176968.396540:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176968.396565:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176968.396570:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176968.396573:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176968.396575:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176968.396577:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176968.396579:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176968.780459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176968.780467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176968.780478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176968.780479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176968.780483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176968.780484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176968.780561:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176968.780565:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176968.780569:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176969.420482:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176969.420491:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176969.420494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176969.420500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176969.420505:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176969.420509:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176969.420513:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176969.420516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176969.420520:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176969.740495:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306316545, at 4306316608 00000100:00000010:0.0:1786176969.740517:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62480. 00010000:00000001:0.0:1786176969.804476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176969.804481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176969.804487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176969.804491:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176969.804496:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176969.804500:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176969.804504:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176969.804507:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176969.804512:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176969.804514:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176969.804520:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176969.804522:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176969.804585:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176969.804589:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176969.804592:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176970.444472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176970.444481:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176970.444484:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176970.444490:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176970.444494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176970.444498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176970.444501:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176970.444504:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176970.444507:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176970.828476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176970.828480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176970.828487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176970.828490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176970.828495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176970.828497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176970.828555:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176970.828559:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176970.828561:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786176970.956466:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176970.956477:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786176970.956480:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176970.956484:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176970.956488:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176970.956497:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11645 00000400:00000200:0.0:1786176970.956506:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176970.956512:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176970.956516:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176970.956533:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.956536:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176970.956539:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.956542:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176970.956546:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176970.956548:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786176970.956552:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176970.956555:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786176970.956558:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786176970.956560:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786176970.956563:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.956566:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176970.956568:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.956570:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176970.956574:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786176970.956577:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176970.956581:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176970.956584:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176970.956587:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176970.956591:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.956601:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786176970.956604:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176970.956615:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786176970.956618:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786176970.956623:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176970.956625:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.956628:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176970.956631:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176970.956636:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176970.956646:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786176970.956652:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176970.956671:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235559040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176970.956682:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176970.956686:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176970.956691:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235559040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176970.956717:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176970.956721:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176970.956724:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786176970.956726:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176970.956728:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176970.956732:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176970.956735:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11645 00000400:00000200:0.0:1786176970.956739:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176970.956743:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176970.956746:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176970.956750:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.956753:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176970.956755:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.956759:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176970.956765:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176970.956767:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786176970.956771:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176970.956774:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786176970.956778:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786176970.956781:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786176970.956787:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.956790:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176970.956792:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.956794:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176970.956799:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786176970.956805:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176970.956811:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176970.956815:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176970.956822:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176970.956828:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.956833:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176970.956839:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176970.956846:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.956849:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176970.956855:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786176970.956861:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176970.956866:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176970.956868:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.956873:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176970.956877:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176970.956881:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176970.956888:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786176970.956895:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176970.956904:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235559168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176970.956948:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176970.956952:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176970.956959:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235559168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176970.956977:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176970.956983:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176970.956991:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786176970.956995:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176970.956998:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176970.957004:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176970.957009:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11645 00000400:00000200:0.0:1786176970.957024:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176970.957031:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176970.957036:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176970.957042:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:1.1:1786176970.957044:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786176970.957052:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176970.957055:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957057:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176970.957060:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176970.957064:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786176970.957067:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176970.957070:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786176970.957075:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786176970.957079:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786176970.957085:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957087:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176970.957092:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957095:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176970.957098:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786176970.957104:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957110:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176970.957113:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176970.957119:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176970.957126:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.957133:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176970.957136:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786176970.957142:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.957147:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176970.957155:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786176970.957162:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786176970.957168:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176970.957172:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.957177:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176970.957181:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176970.957185:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176970.957194:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786176970.957199:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176970.957211:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235559296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176970.957226:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176970.957232:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176970.957241:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235559296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176970.957261:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176970.957266:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176970.957269:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786176970.957270:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176970.957272:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176970.957274:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176970.957279:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11645 00000400:00000200:0.0:1786176970.957287:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176970.957291:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176970.957294:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176970.957299:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957302:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176970.957307:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957311:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176970.957315:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176970.957317:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786176970.957321:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176970.957324:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786176970.957327:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786176970.957332:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786176970.957340:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957344:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176970.957349:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957351:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176970.957355:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786176970.957381:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957384:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176970.957386:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176970.957390:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176970.957392:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.957394:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176970.957396:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176970.957402:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.957404:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176970.957408:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786176970.957413:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176970.957420:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176970.957423:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.957428:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1786176970.957433:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:0.0:1786176970.957434:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176970.957438:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176970.957443:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786176970.957447:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176970.957454:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235559424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176970.957468:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176970.957473:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176970.957482:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235559424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176970.957503:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786176970.957508:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786176970.957513:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786176970.957530:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786176970.957533:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786176970.957535:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786176970.957538:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11645 00000400:00000200:0.0:1786176970.957542:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786176970.957546:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786176970.957548:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786176970.957551:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957553:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176970.957555:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957557:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786176970.957559:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786176970.957560:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786176970.957562:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786176970.957563:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786176970.957565:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786176970.957568:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786176970.957573:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957576:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786176970.957578:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957583:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786176970.957588:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786176970.957595:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786176970.957601:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786176970.957605:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786176970.957610:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786176970.957615:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.957620:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786176970.957623:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786176970.957628:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.957632:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786176970.957639:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786176970.957644:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786176970.957651:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786176970.957653:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786176970.957658:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786176970.957661:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786176970.957666:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786176970.957672:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786176970.957677:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176970.957686:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235559552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786176970.957695:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786176970.957698:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176970.957707:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235559552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786176970.957725:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176970.957733:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11655) 00000800:00000200:1.1:1786176970.957839:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176970.957981:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176970.958143:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786176971.468646:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176971.468657:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176971.468659:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176971.468664:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176971.468668:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176971.468672:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176971.468674:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176971.468677:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176971.468680:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176971.852501:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176971.852524:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176971.852527:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176971.852531:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176971.852542:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176971.852546:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176971.852550:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176971.852561:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176971.852568:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176971.852570:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176971.852575:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176971.852578:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176971.852580:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176971.852582:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176971.852584:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176971.852586:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176971.852588:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176971.852590:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176971.852667:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176971.852671:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176971.852674:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176972.492458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176972.492470:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176972.492474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176972.492482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176972.492489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176972.492494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176972.492499:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176972.492515:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176972.492521:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176972.876546:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176972.876555:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176972.876561:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176972.876562:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176972.876565:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176972.876567:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176972.876629:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176972.876633:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176972.876637:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176973.516538:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176973.516548:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176973.516551:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176973.516557:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176973.516561:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176973.516565:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176973.516568:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176973.516571:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176973.516574:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176973.900451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176973.900457:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176973.900462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176973.900463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176973.900466:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176973.900468:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176973.900548:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176973.900553:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176973.900557:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1786176974.028774:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:3.0:1786176974.028832:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1786176974.028845:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176974.028851:0:2381:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:3.0:1786176974.028857:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176974.028863:0:2381:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1786176974.028870:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176974.028876:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:3.0:1786176974.028884:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176974.028890:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176974.028899:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e7f8 00000400:00000010:3.0:1786176974.028903:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e7f8. 00000100:00000001:3.0:1786176974.028909:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176974.028921:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09315500 x1872930235559040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176974.028954:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176974.028960:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315500 x1872930235559040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786176995 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176974.028969:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176974.028972:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176974.028984:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176974.028988:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:3.0:1786176974.028991:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176974.028996:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176974.029000:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:3.0:1786176974.029002:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:3.0:1786176974.029005:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176974.029011:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09314a80 x1872930235559168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176974.029021:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176974.029027:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314a80 x1872930235559168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176995 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176974.029036:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176974.029040:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176974.029048:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176974.029054:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:3.0:1786176974.029059:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176974.029070:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176974.029094:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e3b8 00000400:00000010:3.0:1786176974.029101:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e3b8. 00000100:00000001:3.0:1786176974.029107:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176974.029117:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09314700 x1872930235559296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176974.029141:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176974.029148:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314700 x1872930235559296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786176995 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176974.029163:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176974.029167:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176974.029174:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176974.029177:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:3.0:1786176974.029181:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176974.029189:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176974.029195:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:3.0:1786176974.029199:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:3.0:1786176974.029203:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176974.029212:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09317480 x1872930235559424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176974.029233:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176974.029240:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317480 x1872930235559424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176995 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176974.029252:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176974.029255:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786176974.029262:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786176974.029267:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:3.0:1786176974.029272:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786176974.029284:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786176974.029288:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e2a8 00000400:00000010:3.0:1786176974.029292:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e2a8. 00000100:00000001:3.0:1786176974.029298:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786176974.029310:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09316d80 x1872930235559552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176995 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176974.029335:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786176974.029343:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316d80 x1872930235559552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786176995 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176974.029410:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176974.029415:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786176974.540472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176974.540514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176974.540518:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176974.540525:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176974.540530:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176974.540534:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176974.540538:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176974.540542:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176974.540546:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176974.860527:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306321664, at 4306321728 00000100:00000010:0.0:1786176974.860550:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62780. 00010000:00000001:0.0:1786176974.924528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176974.924535:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176974.924544:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176974.924548:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176974.924553:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176974.924557:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176974.924621:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176974.924625:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176974.924628:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176975.564523:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176975.564536:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176975.564539:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176975.564548:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176975.564553:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176975.564559:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176975.564564:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176975.564568:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176975.564572:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176975.948458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176975.948463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176975.948468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176975.948484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176975.948490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176975.948494:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176975.948553:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176975.948559:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176975.948562:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176976.076584:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176976.076596:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176976.076598:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176976.076602:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176976.076607:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176976.076615:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11650 00000400:00000200:3.0:1786176976.076624:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176976.076630:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176976.076633:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176976.076635:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.076639:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176976.076641:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.076643:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176976.076647:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176976.076649:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176976.076651:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176976.076654:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176976.076656:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176976.076657:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176976.076659:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.076660:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176976.076662:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.076663:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176976.076666:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176976.076669:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176976.076671:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176976.076674:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176976.076678:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176976.076681:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.076684:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176976.076687:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176976.076695:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176976.076700:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176976.076704:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176976.076707:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.076710:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176976.076715:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176976.076720:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176976.076730:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786176976.076739:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176976.076753:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235559680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176976.076762:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176976.076767:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176976.076773:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235559680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176976.076800:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176976.076804:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176976.076809:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176976.076811:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176976.076813:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176976.076816:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176976.076819:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11650 00000400:00000200:3.0:1786176976.076823:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176976.076826:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176976.076829:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176976.076834:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.076837:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176976.076840:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.076843:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176976.076846:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176976.076848:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176976.076850:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176976.076852:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176976.076856:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176976.076859:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176976.076863:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.076866:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176976.076868:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.076871:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176976.076873:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176976.076876:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176976.076879:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176976.076882:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176976.076892:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176976.076899:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.076904:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176976.076916:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176976.076931:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.076936:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176976.076943:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176976.076948:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176976.076951:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176976.076952:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.076955:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176976.076958:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176976.076961:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176976.076971:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786176976.076984:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176976.076997:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235559808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176976.077018:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176976.077025:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176976.077042:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235559808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176976.077062:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176976.077073:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176976.077083:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176976.077084:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176976.077086:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176976.077090:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176976.077102:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11650 00000400:00000200:3.0:1786176976.077114:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176976.077122:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176976.077128:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176976.077139:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.077143:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176976.077148:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.077153:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176976.077159:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176976.077166:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176976.077173:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176976.077179:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176976.077184:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176976.077191:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176976.077203:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.077214:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176976.077222:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.077228:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176976.077235:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176976.077246:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176976.077254:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176976.077259:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176976.077266:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176976.077272:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.077277:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176976.077282:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176976.077285:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.077287:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176976.077298:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176976.077304:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176976.077311:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176976.077315:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.077320:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176976.077324:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176976.077329:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176976.077487:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176976.077492:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 02000000:00000010:3.0:1786176976.077499:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786176976.077510:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176976.077522:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235559936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176976.077543:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176976.077551:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176976.077564:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235559936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176976.077613:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176976.077620:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176976.077628:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176976.077633:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176976.077638:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176976.077653:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176976.077661:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11650 00000400:00000200:3.0:1786176976.077673:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176976.077684:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176976.077692:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176976.077702:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.077709:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176976.077714:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.077720:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176976.077726:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176976.077730:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176976.077733:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176976.077738:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176976.077743:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176976.077746:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176976.077752:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.077758:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176976.077765:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.077771:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176976.077778:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176976.077787:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176976.077793:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176976.077798:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176976.077805:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176976.077816:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.077823:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176976.077829:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176976.077840:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.077844:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176976.077857:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176976.077864:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176976.077872:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176976.077877:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.077883:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176976.077888:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176976.077895:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176976.077908:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786176976.077915:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176976.077929:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235560064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176976.077965:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176976.077972:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176976.077984:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235560064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176976.078004:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176976.078013:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176976.078028:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176976.078034:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176976.078038:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176976.078047:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176976.078055:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11650 00000400:00000200:3.0:1786176976.078070:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176976.078081:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176976.078088:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176976.078098:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.078104:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176976.078112:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.078119:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176976.078126:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176976.078131:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176976.078138:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176976.078143:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176976.078151:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176976.078159:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176976.078185:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.078193:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176976.078201:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176976.078208:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176976.078218:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176976.078229:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176976.078240:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176976.078246:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176976.078255:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176976.078264:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.078268:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176976.078274:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176976.078282:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.078288:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176976.078301:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176976.078310:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176976.078317:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176976.078321:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176976.078329:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176976.078333:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176976.078338:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176976.078347:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176976.078355:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176976.078407:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235560192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176976.078424:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176976.078430:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176976.078443:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235560192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176976.078480:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176976.078491:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11660) 00000800:00000200:1.1:1786176976.078642:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176976.078645:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176976.078954:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176976.078956:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176976.079274:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176976.079277:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786176976.079760:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176976.079763:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1786176976.588475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176976.588486:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176976.588489:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176976.588496:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176976.588503:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176976.588507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176976.588511:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176976.588517:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176976.588522:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176976.972487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176976.972495:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176976.972502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176976.972503:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176976.972507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176976.972508:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176976.972589:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176976.972593:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176976.972596:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176977.612490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176977.612500:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176977.612503:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176977.612510:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176977.612514:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176977.612518:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176977.612522:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176977.612525:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176977.612529:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176977.996468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176977.996473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176977.996480:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176977.996482:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176977.996487:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176977.996491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176977.996563:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176977.996566:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176977.996569:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176978.636435:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176978.636460:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176978.636464:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176978.636472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176978.636479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176978.636485:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176978.636490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176978.636494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176978.636500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176979.020513:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176979.020521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176979.020529:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176979.020534:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176979.020545:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176979.020550:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176979.020629:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176979.020634:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176979.020637:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176979.660451:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176979.660458:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176979.660461:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176979.660465:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176979.660475:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176979.660478:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176979.660481:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176979.660483:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176979.660486:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176979.980451:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306326785, at 4306326848 00000100:00000010:0.0:1786176979.980468:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62f00. 00010000:00000001:0.0:1786176980.044461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176980.044467:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176980.044475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176980.044481:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176980.044484:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176980.044488:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176980.044491:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176980.044494:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176980.044497:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176980.044500:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176980.044503:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176980.044505:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176980.044526:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176980.044529:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176980.044532:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176980.684513:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176980.684525:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176980.684527:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176980.684534:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176980.684539:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176980.684543:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176980.684546:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176980.684550:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176980.684554:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176981.068512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176981.068517:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176981.068525:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176981.068526:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176981.068530:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176981.068532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176981.068601:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176981.068605:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176981.068609:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786176981.196505:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176981.196521:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786176981.196524:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176981.196529:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176981.196535:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176981.196544:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11655 00000400:00000200:2.0:1786176981.196556:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176981.196564:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176981.196568:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176981.196572:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196576:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176981.196579:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196582:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176981.196588:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176981.196590:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786176981.196593:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176981.196597:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786176981.196600:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786176981.196604:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786176981.196608:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196610:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176981.196612:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196614:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176981.196617:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786176981.196628:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196630:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176981.196633:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176981.196636:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176981.196639:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.196640:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786176981.196643:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176981.196654:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 00000020:00000040:2.0:1786176981.196657:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786176981.196661:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176981.196663:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.196666:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176981.196669:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176981.196674:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176981.196682:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:2.0:1786176981.196688:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176981.196698:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235560320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176981.196708:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176981.196712:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176981.196717:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235560320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176981.196736:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176981.196738:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176981.196741:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786176981.196743:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176981.196744:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176981.196747:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176981.196749:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11655 00000400:00000200:2.0:1786176981.196752:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176981.196754:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176981.196756:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176981.196762:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196765:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176981.196767:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196769:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176981.196772:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176981.196773:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786176981.196775:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176981.196779:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786176981.196781:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786176981.196784:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786176981.196788:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196791:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176981.196797:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196799:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176981.196804:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786176981.196810:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196815:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176981.196818:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176981.196826:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176981.196833:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.196837:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176981.196844:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176981.196852:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.196857:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176981.196864:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786176981.196870:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176981.196875:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176981.196878:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.196880:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176981.196882:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176981.196884:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176981.196891:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aabc00. 00000100:00000001:2.0:1786176981.196897:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176981.196908:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235560448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176981.196920:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176981.196925:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176981.196932:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235560448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176981.196945:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176981.196949:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176981.196955:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786176981.196957:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176981.196958:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176981.196960:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176981.196963:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11655 00000400:00000200:2.0:1786176981.196970:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176981.196972:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176981.196974:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176981.196976:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196978:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176981.196982:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.196986:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176981.196990:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176981.196992:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786176981.196995:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176981.196998:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786176981.197000:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786176981.197001:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786176981.197006:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197010:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176981.197013:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197017:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176981.197021:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786176981.197026:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197030:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176981.197032:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176981.197036:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176981.197039:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.197042:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176981.197045:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786176981.197049:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.197053:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176981.197066:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786176981.197070:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786176981.197074:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176981.197076:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.197077:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176981.197078:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176981.197079:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176981.197083:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9000. 00000100:00000001:2.0:1786176981.197088:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176981.197094:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235560576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176981.197105:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176981.197107:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176981.197113:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235560576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176981.197122:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176981.197125:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176981.197128:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786176981.197131:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176981.197134:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176981.197138:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176981.197143:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11655 00000400:00000200:2.0:1786176981.197148:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176981.197152:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176981.197156:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176981.197161:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197164:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176981.197168:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197173:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176981.197174:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176981.197175:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786176981.197176:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176981.197178:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786176981.197181:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786176981.197184:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786176981.197191:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197195:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176981.197200:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197204:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176981.197209:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786176981.197216:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197221:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176981.197227:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176981.197229:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176981.197232:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.197237:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176981.197241:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176981.197245:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.197250:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176981.197257:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000800:00000200:1.1:1786176981.197257:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:2.0:1786176981.197262:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000800:00000200:1.1:1786176981.197266:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 02000000:00000001:2.0:1786176981.197267:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176981.197270:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.197274:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176981.197278:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176981.197281:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176981.197288:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:2.0:1786176981.197292:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176981.197299:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235560704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176981.197316:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176981.197319:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176981.197328:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235560704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176981.197339:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786176981.197344:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786176981.197352:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786176981.197355:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786176981.197406:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786176981.197411:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786176981.197416:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11655 00000400:00000200:2.0:1786176981.197437:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786176981.197442:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786176981.197444:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786176981.197447:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197449:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176981.197451:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197455:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786176981.197457:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786176981.197459:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786176981.197461:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786176981.197464:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786176981.197467:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786176981.197469:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786176981.197475:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197476:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786176981.197477:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197479:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786176981.197483:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786176981.197486:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786176981.197487:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786176981.197488:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786176981.197491:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786176981.197495:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.197497:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786176981.197499:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786176981.197500:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.197502:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786176981.197507:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734700. 00000020:00000040:2.0:1786176981.197510:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786176981.197513:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786176981.197514:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786176981.197516:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786176981.197519:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786176981.197522:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786176981.197526:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000001:2.0:1786176981.197532:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786176981.197540:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734700 x1872930235560832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786176981.197549:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786176981.197553:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176981.197558:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088313762] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734700 x1872930235560832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786176981.197580:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786176981.197586:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11665) 00000800:00000200:1.1:1786176981.197947:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176981.197950:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176981.198256:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176981.198259:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176981.198509:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176981.198511:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786176981.198728:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176981.198731:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1786176981.708522:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176981.708534:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176981.708537:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176981.708544:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176981.708549:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176981.708554:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176981.708558:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176981.708561:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176981.708565:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176982.092522:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176982.092555:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176982.092559:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176982.092564:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176982.092570:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176982.092574:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176982.092577:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176982.092580:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176982.092584:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176982.092585:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176982.092588:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176982.092590:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176982.092591:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176982.092593:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176982.092594:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176982.092595:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176982.092597:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176982.092600:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176982.092651:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176982.092655:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176982.092659:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176982.732454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176982.732467:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176982.732470:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176982.732478:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176982.732485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176982.732490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176982.732494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176982.732498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176982.732503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176983.116535:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176983.116542:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176983.116557:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176983.116562:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176983.116571:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176983.116577:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176983.116657:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176983.116662:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176983.116666:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176983.756467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176983.756476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176983.756478:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176983.756493:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176983.756497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176983.756502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176983.756506:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176983.756510:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176983.756515:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176984.140458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176984.140463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176984.140468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176984.140470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176984.140477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176984.140480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786176984.140498:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786176984.140501:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786176984.140503:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176984.780469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176984.780483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176984.780488:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176984.780494:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176984.780498:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176984.780501:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176984.780504:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176984.780505:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176984.780508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176985.100471:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306331905, at 4306331968 00000100:00000010:0.0:1786176985.100489:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62360. 00010000:00000001:0.0:1786176985.164483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176985.164488:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176985.164496:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176985.164499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176985.164506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176985.164509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176985.164572:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176985.164576:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176985.164579:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176985.804485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176985.804492:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176985.804494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176985.804498:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176985.804501:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176985.804504:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176985.804506:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176985.804509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176985.804512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176986.188459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176986.188464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176986.188471:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176986.188475:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176986.188481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176986.188484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176986.316506:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176986.316518:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176986.316521:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176986.316525:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176986.316529:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176986.316537:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11660 00000400:00000200:3.0:1786176986.316544:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176986.316550:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176986.316552:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176986.316555:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316557:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176986.316559:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316561:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176986.316564:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176986.316565:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176986.316568:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176986.316571:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176986.316573:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176986.316574:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176986.316577:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316578:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176986.316579:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316580:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176986.316583:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176986.316585:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316586:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176986.316589:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176986.316591:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176986.316593:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.316595:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176986.316597:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176986.316606:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786176986.316608:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176986.316611:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176986.316612:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.316614:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176986.316617:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176986.316621:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176986.316627:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176986.316632:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176986.316639:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235560960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176986.316647:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176986.316650:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176986.316653:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235560960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176986.316670:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176986.316673:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176986.316675:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176986.316676:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176986.316678:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176986.316679:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176986.316681:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11660 00000400:00000200:3.0:1786176986.316683:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176986.316685:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176986.316687:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176986.316690:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316691:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176986.316693:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316696:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176986.316697:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176986.316699:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176986.316700:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176986.316702:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176986.316703:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176986.316705:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176986.316710:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316713:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176986.316717:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316720:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176986.316723:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176986.316727:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316732:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176986.316736:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176986.316742:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176986.316746:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.316749:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176986.316755:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176986.316760:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.316765:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176986.316770:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786176986.316776:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176986.316782:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176986.316793:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.316798:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176986.316801:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176986.316804:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176986.316812:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176986.316819:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176986.316830:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235561088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176986.316857:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176986.316862:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176986.316868:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235561088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176986.316881:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176986.316884:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176986.316886:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176986.316889:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176986.316893:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176986.316895:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176986.316901:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11660 00000400:00000200:3.0:1786176986.316907:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176986.316913:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176986.316916:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176986.316922:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316926:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176986.316933:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316937:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176986.316941:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176986.316943:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176986.316944:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176986.316947:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176986.316952:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176986.316957:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176986.316963:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316966:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176986.316968:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316972:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176986.316974:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176986.316979:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176986.316983:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176986.316985:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176986.316989:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176986.316994:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176986.316994:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176986.316999:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:1.1:1786176986.317000:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000008:00100000:3.0:1786176986.317001:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176986.317007:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.317012:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176986.317019:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786176986.317022:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176986.317028:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176986.317031:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.317033:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176986.317036:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176986.317040:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176986.317044:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176986.317048:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176986.317057:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235561216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176986.317083:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176986.317087:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176986.317095:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235561216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176986.317109:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176986.317112:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176986.317118:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176986.317124:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176986.317129:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176986.317135:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176986.317141:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11660 00000400:00000200:3.0:1786176986.317148:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176986.317154:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176986.317162:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176986.317171:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.317176:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176986.317181:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.317186:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176986.317189:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176986.317192:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176986.317195:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176986.317198:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176986.317202:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176986.317206:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176986.317212:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.317217:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176986.317221:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.317226:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176986.317231:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176986.317238:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176986.317244:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176986.317246:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176986.317250:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176986.317255:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.317258:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176986.317261:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176986.317267:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.317271:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176986.317277:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786176986.317280:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176986.317285:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176986.317287:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.317291:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176986.317294:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176986.317297:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176986.317302:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176986.317307:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176986.317316:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235561344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176986.317331:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176986.317335:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176986.317341:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235561344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176986.317355:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176986.317395:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176986.317398:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176986.317399:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176986.317400:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176986.317402:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176986.317406:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11660 00000400:00000200:3.0:1786176986.317412:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176986.317416:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176986.317421:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176986.317427:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.317430:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176986.317434:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.317437:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176986.317441:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176986.317444:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176986.317449:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176986.317454:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176986.317458:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176986.317461:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176986.317465:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.317469:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176986.317474:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176986.317477:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176986.317479:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176986.317481:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176986.317483:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176986.317487:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176986.317491:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176986.317495:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.317498:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176986.317501:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176986.317505:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.317510:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176986.317517:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786176986.317521:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176986.317526:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176986.317529:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176986.317534:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176986.317537:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176986.317539:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176986.317542:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176986.317544:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176986.317551:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235561472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176986.317562:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176986.317566:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176986.317572:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235561472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176986.317584:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176986.317590:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11670) 00000800:00000200:1.1:1786176986.317649:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176986.317651:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176986.317834:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176986.317836:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176986.318009:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176986.318010:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786176986.318261:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176986.318263:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1786176986.828490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176986.828498:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176986.828501:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176986.828508:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176986.828512:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176986.828516:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176986.828520:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176986.828522:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176986.828525:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176987.212464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176987.212469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176987.212475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176987.212478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176987.212483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176987.212486:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176987.212540:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176987.212544:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176987.212547:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176987.852470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176987.852478:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176987.852480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176987.852484:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176987.852488:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176987.852490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176987.852493:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176987.852495:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176987.852498:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176988.236458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176988.236463:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176988.236468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176988.236471:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176988.236475:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176988.236478:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176988.236546:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176988.236551:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176988.236554:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176988.876461:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176988.876468:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176988.876470:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176988.876475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176988.876479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176988.876482:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176988.876484:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176988.876487:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176988.876489:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176989.260472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176989.260476:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176989.260482:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176989.260484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176989.260489:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176989.260491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176989.260548:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176989.260551:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176989.260554:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176989.900479:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176989.900487:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176989.900490:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176989.900496:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176989.900501:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176989.900506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176989.900508:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176989.900511:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176989.900515:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176990.220476:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306337024, at 4306337088 00000100:00000010:0.0:1786176990.220495:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62a20. 00010000:00000001:0.0:1786176990.284459:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176990.284464:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176990.284472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176990.284478:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176990.284483:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176990.284489:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176990.284496:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176990.284501:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176990.284504:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176990.284507:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176990.284512:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176990.284514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176990.284575:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176990.284578:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176990.284581:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176990.924519:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176990.924526:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176990.924529:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176990.924535:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176990.924540:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176990.924544:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176990.924547:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176990.924551:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176990.924554:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176991.308478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176991.308484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176991.308490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176991.308493:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176991.308497:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176991.308499:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176991.308563:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176991.308568:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176991.308572:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176991.436461:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176991.436469:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176991.436472:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176991.436475:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176991.436479:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176991.436485:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11665 00000400:00000200:3.0:1786176991.436492:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176991.436498:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176991.436501:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176991.436503:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436506:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176991.436508:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436510:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176991.436513:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176991.436515:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176991.436517:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176991.436526:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176991.436528:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176991.436530:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176991.436532:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436534:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176991.436535:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436537:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176991.436539:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176991.436541:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436542:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176991.436544:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176991.436547:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176991.436549:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.436551:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176991.436553:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176991.436561:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786176991.436563:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176991.436566:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176991.436567:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.436570:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176991.436572:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176991.436575:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176991.436582:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786176991.436587:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176991.436595:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235561600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176991.436602:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176991.436606:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176991.436609:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235561600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176991.436626:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176991.436628:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176991.436631:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176991.436632:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176991.436634:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176991.436636:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176991.436637:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11665 00000400:00000200:3.0:1786176991.436640:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176991.436642:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176991.436643:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176991.436645:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436647:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176991.436649:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436650:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176991.436652:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176991.436653:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176991.436655:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176991.436657:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176991.436658:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176991.436660:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176991.436662:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436663:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176991.436665:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436666:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176991.436667:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176991.436670:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436674:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176991.436675:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176991.436681:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176991.436686:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.436688:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176991.436691:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176991.436696:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.436701:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176991.436705:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786176991.436710:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176991.436712:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176991.436715:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.436717:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176991.436720:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176991.436722:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176991.436725:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786176991.436729:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176991.436739:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235561728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176991.436753:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176991.436756:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176991.436762:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235561728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176991.436772:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176991.436778:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176991.436780:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176991.436782:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176991.436785:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176991.436788:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176991.436792:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11665 00000400:00000200:3.0:1786176991.436795:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176991.436799:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176991.436801:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176991.436803:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436809:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176991.436814:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436818:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176991.436820:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176991.436823:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176991.436826:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176991.436829:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176991.436832:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176991.436837:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176991.436849:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436852:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176991.436854:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436858:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176991.436863:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176991.436867:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176991.436871:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176991.436874:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176991.436877:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176991.436882:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.436884:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176991.436888:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176991.436893:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.436896:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176991.436901:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786176991.436907:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176991.436912:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176991.436914:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.436917:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176991.436919:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176991.436921:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176991.436926:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786176991.436931:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176991.436937:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235561856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176991.436946:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176991.436949:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176991.436956:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235561856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1786176991.436969:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176991.436970:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176991.436972:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00080000:3.0:1786176991.436973:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176991.436979:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176991.436981:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176991.436984:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176991.436994:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176991.437000:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11665 00000400:00000200:3.0:1786176991.437004:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176991.437010:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176991.437013:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176991.437018:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.437021:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176991.437026:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.437030:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176991.437033:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176991.437036:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176991.437039:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176991.437041:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176991.437044:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176991.437047:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176991.437055:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.437059:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176991.437063:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.437071:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176991.437078:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176991.437088:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176991.437093:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176991.437104:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176991.437110:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176991.437118:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.437125:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176991.437129:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176991.437135:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.437138:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176991.437141:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786176991.437143:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176991.437145:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176991.437146:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.437147:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176991.437149:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176991.437152:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176991.437158:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786176991.437162:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176991.437170:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235561984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176991.437186:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176991.437191:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176991.437199:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235561984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176991.437210:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176991.437213:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176991.437221:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176991.437225:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176991.437229:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176991.437234:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176991.437239:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11665 00000400:00000200:3.0:1786176991.437247:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176991.437253:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176991.437258:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176991.437264:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.437268:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176991.437272:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.437276:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176991.437281:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176991.437283:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176991.437286:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176991.437289:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176991.437294:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176991.437298:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176991.437303:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.437308:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176991.437312:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176991.437315:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176991.437318:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176991.437382:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176991.437384:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176991.437386:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176991.437389:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176991.437391:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.437393:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176991.437395:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176991.437400:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.437407:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176991.437414:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786176991.437420:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176991.437427:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176991.437431:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176991.437435:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176991.437439:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176991.437444:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176991.437451:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786176991.437458:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176991.437465:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235562112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176991.437476:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176991.437479:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176991.437485:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235562112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176991.437495:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176991.437499:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11676) 00000800:00000200:1.1:1786176991.437634:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176991.437637:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176991.437896:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176991.437898:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176991.438079:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176991.438081:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786176991.438255:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176991.438257:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00000001:0.0:1786176991.948560:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176991.948566:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176991.948568:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176991.948572:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176991.948575:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176991.948578:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176991.948580:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176991.948582:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176991.948584:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176992.332484:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176992.332492:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176992.332495:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176992.332499:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176992.332504:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176992.332507:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176992.332510:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176992.332513:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176992.332520:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176992.332523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176992.332527:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176992.332530:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176992.332532:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176992.332533:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176992.332534:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176992.332535:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176992.332537:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176992.332538:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176992.332588:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176992.332592:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176992.332595:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176992.972467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176992.972474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176992.972476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176992.972481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176992.972485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176992.972489:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176992.972492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176992.972494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176992.972496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176993.356492:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176993.356497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176993.356507:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176993.356511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176993.356520:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176993.356523:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176993.356585:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176993.356590:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176993.356594:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176993.996431:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176993.996438:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176993.996440:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176993.996447:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176993.996451:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176993.996456:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176993.996459:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176993.996462:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176993.996465:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176994.380468:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176994.380473:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176994.380483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176994.380487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176994.380494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176994.380497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176994.380563:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176994.380568:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176994.380572:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176995.020468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176995.020477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176995.020480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176995.020486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176995.020490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176995.020495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176995.020498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176995.020501:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176995.020505:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786176995.340431:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306342144, at 4306342208 00000100:00000010:0.0:1786176995.340446:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62300. 00010000:00000001:0.0:1786176995.404464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176995.404469:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176995.404476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176995.404480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176995.404488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176995.404492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176995.404546:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176995.404550:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176995.404553:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176996.044430:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176996.044438:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176996.044439:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176996.044445:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176996.044449:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176996.044453:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176996.044455:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176996.044457:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176996.044459:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176996.428477:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176996.428483:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176996.428491:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176996.428496:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176996.428505:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176996.428509:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176996.428576:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176996.428581:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176996.428585:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786176996.556459:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176996.556467:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786176996.556469:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176996.556472:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176996.556477:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176996.556484:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11671 00000400:00000200:3.0:1786176996.556493:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176996.556500:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176996.556503:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176996.556507:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556511:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176996.556514:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556517:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176996.556521:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176996.556523:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786176996.556526:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176996.556530:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786176996.556533:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786176996.556535:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786176996.556539:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556540:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176996.556542:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556543:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176996.556546:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786176996.556548:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556550:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176996.556552:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176996.556554:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176996.556556:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.556557:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786176996.556566:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176996.556575:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786176996.556578:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786176996.556582:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176996.556583:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.556585:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176996.556587:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176996.556591:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176996.556597:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786176996.556603:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176996.556610:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235562240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176996.556618:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176996.556622:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176996.556625:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235562240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176996.556645:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176996.556647:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176996.556650:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786176996.556651:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176996.556652:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176996.556654:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176996.556656:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11671 00000400:00000200:3.0:1786176996.556659:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176996.556661:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176996.556663:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176996.556665:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556666:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176996.556668:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556669:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176996.556671:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176996.556672:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786176996.556674:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176996.556676:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786176996.556677:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786176996.556679:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786176996.556681:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556682:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176996.556688:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556690:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176996.556692:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786176996.556694:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556697:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176996.556698:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176996.556702:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176996.556705:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.556708:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176996.556714:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786176996.556717:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.556719:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176996.556723:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786176996.556727:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176996.556732:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176996.556734:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.556742:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176996.556745:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176996.556748:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176996.556753:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786176996.556759:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176996.556769:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235562368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176996.556786:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176996.556792:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176996.556800:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235562368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176996.556820:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176996.556826:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176996.556831:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786176996.556835:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176996.556840:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176996.556845:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176996.556852:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11671 00000400:00000200:3.0:1786176996.556862:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176996.556870:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176996.556875:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176996.556882:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556888:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176996.556893:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556899:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176996.556904:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176996.556907:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786176996.556910:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176996.556916:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786176996.556922:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786176996.556928:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786176996.556936:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556938:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176996.556943:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556947:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176996.556950:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786176996.556955:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176996.556959:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176996.556963:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176996.556965:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176996.556969:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.556974:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176996.556979:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.1:1786176996.556984:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000008:00000001:3.0:1786176996.556985:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176996.556987:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1786176996.556990:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176996.556997:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786176996.557002:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786176996.557009:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176996.557012:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.557015:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176996.557018:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176996.557022:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176996.557027:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786176996.557033:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176996.557043:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235562496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176996.557064:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176996.557068:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176996.557077:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235562496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176996.557093:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176996.557099:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176996.557108:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786176996.557113:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176996.557116:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176996.557123:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176996.557128:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11671 00000400:00000200:3.0:1786176996.557137:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176996.557146:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176996.557151:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176996.557160:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.557165:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176996.557173:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.557181:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176996.557186:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176996.557189:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786176996.557193:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176996.557198:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786176996.557203:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786176996.557209:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786176996.557218:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.557223:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176996.557229:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.557235:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176996.557241:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786176996.557248:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786176996.557255:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786176996.557260:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176996.557267:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176996.557307:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.557312:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176996.557318:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176996.557326:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.557333:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176996.557341:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786176996.557348:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176996.557354:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176996.557387:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.557393:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176996.557395:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176996.557397:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176996.557405:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786176996.557412:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176996.557425:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235562624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176996.557441:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176996.557444:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176996.557450:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235562624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176996.557471:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786176996.557477:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786176996.557488:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786176996.557493:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786176996.557498:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786176996.557507:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786176996.557514:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11671 00000400:00000200:3.0:1786176996.557523:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786176996.557531:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786176996.557537:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786176996.557544:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.557550:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176996.557558:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.557565:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786176996.557572:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786176996.557575:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786176996.557581:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786176996.557586:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786176996.557593:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786176996.557600:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786176996.557610:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.557617:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786176996.557623:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786176996.557628:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786176996.557636:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786176996.557645:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:1.1:1786176996.557649:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786176996.557651:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786176996.557652:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 02000000:00000001:3.0:1786176996.557657:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786176996.557664:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786176996.557671:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.557677:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786176996.557681:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786176996.557689:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.557695:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786176996.557703:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786176996.557710:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786176996.557716:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786176996.557720:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786176996.557725:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786176996.557728:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786176996.557733:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786176996.557741:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786176996.557748:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176996.557759:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235562752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786176996.557797:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786176996.557803:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176996.557817:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235562752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786176996.557845:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786176996.557852:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11681) 00000800:00000200:1.1:1786176996.557979:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176996.557980:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176996.558266:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176996.558268:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1786176996.558491:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786176996.558492:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:0.0:1786176997.068461:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176997.068469:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176997.068471:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176997.068476:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176997.068481:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176997.068483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176997.068486:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176997.068488:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176997.068490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176997.452465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176997.452470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176997.452476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176997.452481:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176997.452488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176997.452491:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176997.452551:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176997.452554:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176997.452556:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786176998.092423:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786176998.092430:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786176998.092432:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786176998.092437:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786176998.092441:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786176998.092445:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176998.092448:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786176998.092450:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786176998.092453:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176998.476467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176998.476472:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176998.476478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176998.476480:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176998.476486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176998.476487:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176998.476559:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176998.476564:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176998.476568:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786176999.116615:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786176999.116622:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786176999.116623:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786176999.116628:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786176999.116631:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786176999.116633:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176999.116635:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786176999.116637:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786176999.116639:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176999.500451:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176999.500455:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176999.500461:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176999.500465:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786176999.500468:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786176999.500471:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176999.500477:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786176999.500479:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176999.500482:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176999.500485:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786176999.500490:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786176999.500492:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786176999.500551:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786176999.500554:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786176999.500557:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177000.140540:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177000.140552:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177000.140555:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177000.140563:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177000.140569:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177000.140575:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177000.140579:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177000.140584:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177000.140588:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177000.460479:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306347264, at 4306347328 00000100:00000010:0.0:1786177000.460492:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c622a0. 00010000:00000001:0.0:1786177000.524506:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177000.524514:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177000.524522:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177000.524524:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177000.524528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177000.524531:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177000.524597:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177000.524601:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177000.524604:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177001.164543:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177001.164553:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177001.164556:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177001.164562:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177001.164568:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177001.164573:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177001.164577:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177001.164581:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177001.164584:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177001.548452:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177001.548462:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177001.548464:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177001.548469:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177001.548475:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177001.548480:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177001.548482:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177001.548487:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177001.548495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177001.548498:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177001.548502:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177001.548506:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177001.548519:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177001.548526:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177001.548537:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177001.548541:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177001.548544:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177001.548547:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177001.548594:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177001.548597:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177001.548600:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177001.676496:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177001.676508:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177001.676511:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177001.676515:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177001.676520:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177001.676528:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11676 00000400:00000200:3.0:1786177001.676538:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177001.676545:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177001.676548:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177001.676552:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.676555:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177001.676558:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.676561:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177001.676565:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177001.676567:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177001.676570:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177001.676574:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177001.676577:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177001.676580:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177001.676583:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.676586:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177001.676588:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.676590:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177001.676593:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177001.676597:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177001.676600:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177001.676603:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177001.676607:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177001.676610:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.676613:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177001.676615:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177001.676626:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000020:00000040:3.0:1786177001.676630:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177001.676634:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177001.676636:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.676640:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177001.676642:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177001.676647:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177001.676655:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786177001.676662:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177001.676673:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235562880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177001.676684:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177001.676689:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177001.676695:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235562880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177001.676720:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177001.676728:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177001.676732:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177001.676734:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177001.676736:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177001.676739:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177001.676741:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11676 00000400:00000200:3.0:1786177001.676746:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177001.676750:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177001.676753:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177001.676757:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.676760:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177001.676763:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.676765:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177001.676768:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177001.676769:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177001.676772:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177001.676777:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177001.676781:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177001.676786:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177001.676792:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.676794:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177001.676796:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.676799:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177001.676802:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177001.676808:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177001.676814:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177001.676819:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177001.676827:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177001.676831:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.676836:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177001.676840:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177001.676847:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.676851:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177001.676858:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786177001.676864:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177001.676869:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177001.676872:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.676875:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177001.676880:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177001.676883:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177001.676891:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786177001.676897:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177001.676907:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235563008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177001.676924:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177001.676928:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177001.676937:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235563008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177001.676972:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177001.676979:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177001.676986:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177001.676992:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177001.676996:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177001.677002:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177001.677008:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11676 00000400:00000200:3.0:1786177001.677016:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177001.677023:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177001.677026:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177001.677035:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677039:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177001.677046:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677052:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177001.677058:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177001.677062:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177001.677066:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177001.677071:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177001.677078:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177001.677083:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177001.677091:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677096:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177001.677099:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677105:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177001.677111:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177001.677119:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677124:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177001.677128:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177001.677134:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177001.677137:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.677142:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177001.677144:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177001.677150:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.677155:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177001.677162:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000800:00000200:1.1:1786177001.677168:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786177001.677169:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177001.677176:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.1:1786177001.677176:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 02000000:00000001:3.0:1786177001.677178:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.677184:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177001.677187:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177001.677191:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177001.677198:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786177001.677204:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177001.677216:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235563136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177001.677245:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177001.677248:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177001.677253:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235563136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177001.677266:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177001.677271:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177001.677278:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177001.677282:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177001.677284:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177001.677292:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177001.677300:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11676 00000400:00000200:3.0:1786177001.677308:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177001.677317:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177001.677322:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177001.677330:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677335:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177001.677340:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677344:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177001.677348:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177001.677351:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177001.677355:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177001.677394:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177001.677398:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177001.677400:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177001.677404:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677408:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177001.677411:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677416:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177001.677421:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177001.677428:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677433:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177001.677438:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177001.677443:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177001.677447:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.677449:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177001.677452:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177001.677458:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.677461:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177001.677469:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786177001.677474:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177001.677480:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177001.677484:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.677488:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177001.677491:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177001.677494:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177001.677501:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786177001.677508:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177001.677519:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235563264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177001.677533:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177001.677539:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177001.677549:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235563264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177001.677565:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177001.677572:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177001.677580:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177001.677585:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177001.677588:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177001.677594:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177001.677600:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11676 00000400:00000200:3.0:1786177001.677609:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177001.677616:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177001.677621:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177001.677628:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677633:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177001.677638:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677640:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177001.677643:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177001.677644:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177001.677646:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177001.677650:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177001.677655:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177001.677661:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177001.677669:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677672:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177001.677676:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677681:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177001.677688:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177001.677697:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177001.677704:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177001.677708:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177001.677714:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177001.677721:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.677726:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177001.677729:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177001.677734:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.677738:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177001.677745:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786177001.677750:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177001.677757:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177001.677760:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177001.677764:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177001.677767:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177001.677770:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177001.677778:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786177001.677783:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177001.677794:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235563392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177001.677805:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177001.677809:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177001.677816:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235563392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177001.677829:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177001.677836:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11686) 00000800:00000200:1.1:1786177001.678030:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177001.678033:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786177001.678343:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177001.678346:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786177001.678655:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177001.678658:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786177001.678967:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177001.678970:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1786177002.188480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177002.188493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177002.188494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177002.188500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177002.188503:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177002.188506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177002.188508:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177002.188511:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177002.188513:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177002.572464:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177002.572470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177002.572476:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177002.572479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177002.572486:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177002.572489:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177002.572550:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177002.572554:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177002.572557:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177003.212463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177003.212470:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177003.212472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177003.212476:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177003.212480:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177003.212483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177003.212485:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177003.212487:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177003.212490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177003.596472:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177003.596477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177003.596483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177003.596485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177003.596488:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177003.596490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177003.596549:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177003.596554:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177003.596558:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177004.236472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177004.236487:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177004.236490:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177004.236496:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177004.236501:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177004.236505:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177004.236509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177004.236512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177004.236516:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177004.620483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177004.620490:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177004.620500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177004.620504:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177004.620511:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177004.620513:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177004.620571:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177004.620574:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177004.620577:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786177005.196645:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1786177005.260465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177005.260472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177005.260474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177005.260479:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177005.260483:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177005.260486:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177005.260488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177005.260491:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177005.260493:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177005.580463:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306352385, at 4306352448 00000100:00000010:0.0:1786177005.580481:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c620c0. 00010000:00000001:0.0:1786177005.644460:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177005.644465:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177005.644473:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177005.644477:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177005.644481:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177005.644484:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177005.644539:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177005.644544:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177005.644548:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177006.284484:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177006.284494:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177006.284497:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177006.284504:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177006.284508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177006.284513:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177006.284516:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177006.284520:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177006.284524:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177006.668500:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177006.668506:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177006.668517:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177006.668520:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177006.668528:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177006.668532:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177006.668606:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177006.668609:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177006.668612:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177006.796496:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177006.796508:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177006.796511:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177006.796515:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177006.796519:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177006.796527:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11681 00000400:00000200:2.0:1786177006.796537:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177006.796543:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177006.796546:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177006.796550:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.796554:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177006.796557:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.796560:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177006.796564:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177006.796566:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177006.796569:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177006.796574:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177006.796576:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177006.796579:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177006.796583:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.796585:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177006.796587:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.796589:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177006.796592:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177006.796596:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177006.796599:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177006.796603:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177006.796607:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177006.796610:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.796612:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177006.796615:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177006.796625:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737480. 00000020:00000040:2.0:1786177006.796629:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177006.796633:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177006.796634:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.796638:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177006.796641:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177006.796646:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177006.796654:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000001:2.0:1786177006.796661:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177006.796672:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737480 x1872930235563520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177006.796683:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177006.796688:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177006.796693:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046bba14a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737480 x1872930235563520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177006.796719:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177006.796722:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177006.796726:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177006.796727:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177006.796729:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177006.796733:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177006.796735:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11681 00000400:00000200:2.0:1786177006.796739:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177006.796742:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177006.796745:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177006.796747:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.796750:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177006.796753:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.796755:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177006.796758:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177006.796759:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177006.796761:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177006.796764:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177006.796766:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177006.796769:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177006.796773:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.796775:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177006.796778:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.796781:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177006.796784:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177006.796787:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177006.796790:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177006.796794:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177006.796801:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177006.796806:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.796811:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177006.796817:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177006.796822:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.796826:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177006.796833:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736d80. 00000020:00000040:2.0:1786177006.796839:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177006.796845:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177006.796850:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.796856:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177006.796860:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177006.796865:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177006.796873:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786177006.796881:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177006.796891:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736d80 x1872930235563648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177006.796908:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177006.796914:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177006.796925:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e4f7686] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736d80 x1872930235563648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177006.796947:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177006.796953:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177006.796958:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177006.796962:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177006.796966:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177006.796974:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177006.796980:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11681 00000400:00000200:2.0:1786177006.796990:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177006.797000:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177006.797006:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177006.797014:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797017:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177006.797024:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797030:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177006.797036:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177006.797039:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177006.797043:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177006.797047:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177006.797054:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177006.797059:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000200:1.1:1786177006.797063:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177006.797067:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000040:2.0:1786177006.797068:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797073:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177006.797081:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797085:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177006.797092:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177006.797099:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797105:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177006.797109:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177006.797115:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177006.797120:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.797125:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177006.797129:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177006.797135:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.797141:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177006.797149:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734700. 00000020:00000040:2.0:1786177006.797154:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177006.797160:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177006.797164:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.797168:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177006.797186:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177006.797191:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177006.797209:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177006.797217:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177006.797227:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734700 x1872930235563776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177006.797244:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177006.797250:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177006.797259:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088313762] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734700 x1872930235563776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177006.797282:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177006.797286:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177006.797294:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177006.797297:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177006.797301:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177006.797307:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177006.797313:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11681 00000400:00000200:2.0:1786177006.797322:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177006.797329:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177006.797333:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177006.797340:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797345:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177006.797350:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797354:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177006.797390:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177006.797393:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177006.797399:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177006.797405:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177006.797410:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177006.797415:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177006.797423:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797429:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177006.797435:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797441:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177006.797447:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177006.797455:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797462:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177006.797467:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177006.797472:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177006.797478:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.797483:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177006.797488:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177006.797494:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.797498:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177006.797505:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000020:00000040:2.0:1786177006.797511:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177006.797516:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177006.797520:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.797525:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177006.797528:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177006.797533:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177006.797539:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:2.0:1786177006.797547:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177006.797556:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235563904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177006.797574:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177006.797579:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177006.797592:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235563904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177006.797615:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177006.797622:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177006.797630:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177006.797635:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177006.797640:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177006.797646:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177006.797653:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11681 00000400:00000200:2.0:1786177006.797663:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177006.797673:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177006.797680:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177006.797689:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797695:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177006.797702:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797708:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177006.797713:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177006.797717:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177006.797721:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177006.797725:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177006.797732:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177006.797738:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177006.797747:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797753:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177006.797760:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797767:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:1.1:1786177006.797771:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:2.0:1786177006.797773:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000800:00000200:1.1:1786177006.797774:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00080000:2.0:1786177006.797779:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177006.797786:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177006.797790:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177006.797796:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177006.797802:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.797807:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177006.797812:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177006.797820:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.797824:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177006.797831:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786177006.797837:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177006.797843:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177006.797847:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177006.797850:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177006.797854:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177006.797860:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177006.797868:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:2.0:1786177006.797876:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177006.797888:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177006.797908:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177006.797912:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177006.797923:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177006.797951:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177006.797960:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11691) 00000800:00000200:1.1:1786177006.798075:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177006.798076:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177006.798319:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177006.798321:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:1.0:1786177006.798455:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177006.798457:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177006.798458:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177006.798460:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177006.798462:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177006.798464:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177006.798466:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735880 x1872930235563904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177031 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177006.798470:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235563904:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177006.798473:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177006.798475:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177006.798479:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177006.798481:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177006.798482:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177006.798483:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177006.798484:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177006.798486:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564032:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177006.798488:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177006.798489:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177006.798490:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177006.798491:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177006.798494:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af000. 02000000:00000001:1.0:1786177006.798495:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177006.798497:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177006.798499:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177006.798501:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177006.798503:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235564032, portal 4 00000100:00000001:1.0:1786177006.798505:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177006.798506:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469622784 : -110371239928832 : ffff9b9e31735c00) 00000100:00000040:1.0:1786177006.798510:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177006.798514:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177006.798516:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177006.798517:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177006.798519:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235564032, offset 0 00000400:00000200:1.0:1786177006.798521:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177006.798524:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177006.798526:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177006.798528:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177006.798530:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177006.798531:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177006.798533:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177006.798535:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426473:426473:256:4294967295] 192.168.202.132@tcp LPNI seq info [426435:426435:8:4294967295] 00000400:00000200:1.0:1786177006.798538:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177006.798542:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177006.798545:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177006.798547:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177006.798550:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177006.798552:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177006.798554:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177006.798556:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177006.798559:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177006.798560:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177006.798562:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177006.798564:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177006.798566:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177006.798568:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177006.798569:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177006.798571:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177031 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177006.798576:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177006.798578:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177031 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177006.798587:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177006.798588:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177006.798590:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177006.798592:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177006.798594:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177006.798596:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306353671 00000100:00000001:1.0:1786177006.798597:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306357504 00000100:00000001:1.0:1786177006.798599:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306353671 00000100:00000001:1.0:1786177006.798601:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306357504 00000100:00000001:1.0:1786177006.798602:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177006.798603:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177006.798604:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177006.798607:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177006/real 1786177006] req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177006.798611:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177006.798612:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177006.798613:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177006.798614:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177006.798615:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177006.798617:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177006.798619:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177006.798624:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177006.798628:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177006.798629:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177006.798630:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177006.798631:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177006.798633:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177006.798638:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177006.798640:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177031 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177006.798645:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177006.798646:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177006.798647:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177006.798648:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177006.798649:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177006.798650:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177006.798651:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177006.798653:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177006.798654:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177006.798656:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177031 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177006.798661:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564032:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177006.798663:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177006.798664:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177006.798666:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735880 x1872930235563904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177031 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177006.798670:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177006.798671:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177006.798672:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177006.798673:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177006.798674:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177006.798675:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177006.798676:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177006.798678:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:1.0:1786177006.798679:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177006.798681:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177006.798682:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735880. 00000100:00000001:1.0:1786177006.798683:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177006.798684:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177006.798685:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177006.798687:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735c00 x1872930235564032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177031 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177006.798691:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177006.798692:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177006.798693:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af000. 02000000:00000001:1.0:1786177006.798694:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177006.798695:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177006.798695:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177006.798696:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177006.798697:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:1.0:1786177006.798698:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177006.798699:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177006.798700:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735c00. 00000100:00000001:1.0:1786177006.798701:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177006.798702:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177006.798703:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177006.798705:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177006.798706:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177006.798707:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177006.798708:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177007.308484:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177007.308497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177007.308499:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177007.308504:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177007.308509:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177007.308512:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177007.308514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177007.308516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177007.308519:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177007.692466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177007.692470:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177007.692474:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177007.692478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177007.692479:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177007.692483:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177007.692484:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177007.692485:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177007.692492:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177007.692498:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177007.692503:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177007.692507:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177007.692510:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177007.692514:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177007.692517:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177007.692551:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177007.692554:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177007.692557:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177008.332532:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177008.332540:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177008.332544:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177008.332550:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177008.332555:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177008.332560:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177008.332563:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177008.332567:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177008.332571:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177008.716492:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177008.716495:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177008.716501:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177008.716501:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177008.716505:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177008.716508:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177008.716510:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177008.716511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177008.716518:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177008.716518:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177008.716521:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177008.716523:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177008.716526:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177008.716529:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177008.716531:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177008.716607:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177008.716612:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177008.716616:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177009.356506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177009.356518:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177009.356522:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177009.356532:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177009.356538:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177009.356545:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177009.356549:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177009.356553:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177009.356557:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177009.740459:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177009.740465:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177009.740467:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177009.740470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177009.740471:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177009.740478:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177009.740479:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177009.740483:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177009.740485:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177009.740487:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177009.740490:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177009.740492:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177009.740493:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177009.740497:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177009.740497:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177009.740501:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177009.740502:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177009.740504:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177009.740507:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177009.740510:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177009.740511:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177009.740598:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177009.740605:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177009.740608:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177010.380425:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177010.380435:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177010.380437:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177010.380442:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177010.380446:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177010.380450:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177010.380453:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177010.380457:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177010.380461:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177010.700460:0:206490:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306357504, at 4306357568 00000100:00000040:1.0:1786177010.700460:0:206554:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306357504, at 4306357568 00000100:00000010:0.0:1786177010.700475:0:206490:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c624e0. 00000100:00000010:1.0:1786177010.700478:0:206554:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ea0. 00000100:00000001:1.0:1786177010.764436:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177010.764443:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177010.764445:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177010.764445:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177010.764451:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177010.764451:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177010.764457:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177010.764458:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177010.764460:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177010.764461:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177010.764466:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177010.764467:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177010.764469:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177010.764470:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177010.764472:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177010.764522:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177010.764530:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177010.764533:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177011.404432:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177011.404439:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177011.404441:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177011.404446:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177011.404449:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177011.404453:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177011.404457:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177011.404461:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177011.404466:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.788457:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177011.788462:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177011.788464:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177011.788467:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177011.788468:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177011.788472:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1786177011.788473:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177011.788475:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.788477:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177011.788479:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177011.788481:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177011.788482:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.788484:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177011.788486:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.788488:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177011.788489:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.788491:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177011.788494:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177011.788497:0:206490:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177011.788499:0:206490:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177011.788500:0:206490:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177011.788501:0:206490:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177011.788502:0:206490:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177011.788504:0:206490:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177011.788505:0:206490:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177011.788506:0:206490:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177011.788507:0:206490:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177011.788564:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177011.788569:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177011.788573:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786177011.916493:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177011.916505:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786177011.916508:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177011.916512:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177011.916517:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177011.916523:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11686 00000400:00000200:0.0:1786177011.916533:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177011.916540:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177011.916544:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177011.916557:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177011.916562:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177011.916564:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177011.916567:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177011.916573:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177011.916575:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786177011.916578:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177011.916581:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786177011.916584:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786177011.916587:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786177011.916590:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177011.916592:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177011.916595:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177011.916597:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177011.916600:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786177011.916605:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177011.916607:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177011.916610:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177011.916613:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177011.916617:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177011.916619:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177011.916622:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177011.916633:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786177011.916637:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786177011.916641:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177011.916642:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177011.916646:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177011.916649:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177011.916653:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177011.916661:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786177011.916668:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177011.916677:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177011.916689:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177011.916693:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177011.916699:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177011.916724:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177011.916727:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177011.916731:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786177011.916733:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177011.916735:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177011.916738:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177011.916741:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11686 00000400:00000200:0.0:1786177011.916745:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177011.916749:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177011.916752:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177011.916756:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177011.916759:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177011.916760:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177011.916766:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1786177011.916767:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177011.916769:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1786177011.916773:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177011.916778:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:1.0:1786177011.916778:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1786177011.916782:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786177011.916786:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177011.916790:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00000001:1.0:1786177011.916791:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1786177011.916796:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177011.916796:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177011.916801:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1786177011.916802:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 02000000:00000001:1.0:1786177011.916803:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177011.916807:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177011.916810:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00100000:1.0:1786177011.916813:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235564160:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177011.916817:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177011.916819:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1786177011.916823:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177011.916823:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177011.916826:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177011.916828:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:1.0:1786177011.916830:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1786177011.916834:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 02000000:00000010:1.0:1786177011.916837:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 00000100:00080000:0.0:1786177011.916841:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:1.0:1786177011.916841:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177011.916847:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177011.916849:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177011.916853:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 02000000:00000001:0.0:1786177011.916854:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1786177011.916860:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 02000000:00000001:0.0:1786177011.916862:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177011.916867:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177011.916867:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235564160, portal 25 00000100:00000001:1.0:1786177011.916871:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177011.916873:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177011.916874:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794208768 : -110371915342848 : ffff9b9e09315c00) 00000008:00100000:0.0:1786177011.916881:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1786177011.916882:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1786177011.916889:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.916892:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1786177011.916894:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177011.916896:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177011.916900:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000010:0.0:1786177011.916903:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000100:00000200:1.0:1786177011.916903:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235564160, offset 0 00000020:00000040:0.0:1786177011.916910:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177011.916913:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 02000000:00000001:0.0:1786177011.916918:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177011.916921:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177011.916924:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177011.916926:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177011.916929:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1786177011.916932:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 02000000:00000001:0.0:1786177011.916933:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177011.916937:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:0.0:1786177011.916941:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000400:00000200:1.0:1786177011.916942:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177011.916947:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1786177011.916948:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177011.916949:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177011.916955:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426474:426474:256:4294967295] 192.168.202.132@tcp LPNI seq info [426436:426436:8:4294967295] 00000100:00100000:0.0:1786177011.916961:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177011.916961:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177011.916975:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:0.0:1786177011.916984:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.0:1786177011.916985:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:0.0:1786177011.916989:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177011.916990:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000100:00000040:0.0:1786177011.916998:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1786177011.916999:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177011.917004:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786177011.917012:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177011.917018:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1786177011.917023:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1786177011.917024:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177011.917027:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000100:00080000:0.0:1786177011.917030:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177011.917034:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:0.0:1786177011.917036:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786177011.917040:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177011.917044:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1786177011.917044:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1786177011.917051:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177011.917054:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000100:00080000:0.0:1786177011.917057:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11686 00000400:00000010:1.0:1786177011.917059:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000400:00000200:0.0:1786177011.917066:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177011.917067:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:0.0:1786177011.917073:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000200:1.0:1786177011.917076:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1786177011.917078:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177011.917087:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177011.917093:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177011.917096:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177011.917097:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177011.917102:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:1.0:1786177011.917105:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1786177011.917106:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177011.917109:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786177011.917113:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177011.917117:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000100:00000001:1.0:1786177011.917134:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.917137:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1786177011.917142:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000001:1.0:1786177011.917145:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786177011.917147:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177011.917152:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:0.0:1786177011.917154:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177011.917159:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177011.917159:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306362624 00000100:00000040:0.0:1786177011.917164:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177011.917168:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000010:1.0:1786177011.917168:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62ea0. 00000020:00000040:0.0:1786177011.917173:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786177011.917179:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177011.917179:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177011.917185:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177011.917188:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177011.917189:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306362624 02000000:00000001:0.0:1786177011.917192:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1786177011.917195:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62600. 02000000:00000001:0.0:1786177011.917197:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.917201:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177011.917202:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177011.917208:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1786177011.917210:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1786177011.917211:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177011.917218:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.917218:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177011.917223:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000100:00000001:1.0:1786177011.917223:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000020:00000040:0.0:1786177011.917228:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00100000:1.0:1786177011.917232:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177011/real 1786177011] req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786177011.917233:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177011.917237:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177011.917242:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177011.917244:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177011.917247:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1786177011.917254:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000010:0.0:1786177011.917255:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000400:00000001:1.0:1786177011.917259:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1786177011.917262:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177011.917263:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177011.917270:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000100:00100000:0.0:1786177011.917274:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177011.917274:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177011.917279:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177011.917287:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177011.917289:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177011.917294:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177011.917304:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177011.917309:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177011.917329:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.917331:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1786177011.917335:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177011.917336:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1786177011.917344:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177011.917345:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177011.917347:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177011.917349:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1786177011.917351:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177011.917393:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177011.917398:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11686 00000100:00100000:1.0:1786177011.917399:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1786177011.917408:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177011.917416:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177011.917418:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1786177011.917421:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177011.917429:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177011.917429:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177011.917436:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177011.917441:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177011.917446:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177011.917449:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1786177011.917451:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177011.917454:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:1.0:1786177011.917455:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1786177011.917458:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177011.917459:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:0.0:1786177011.917462:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:1.0:1786177011.917463:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.917468:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000001:0.0:1786177011.917469:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177011.917473:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1786177011.917474:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00080000:1.0:1786177011.917481:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:0.0:1786177011.917482:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177011.917487:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1786177011.917491:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000040:0.0:1786177011.917493:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177011.917496:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1786177011.917498:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177011.917504:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000040:1.0:1786177011.917507:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1786177011.917511:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177011.917518:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177011.917523:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177011.917528:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1786177011.917530:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235564160:192.168.202.132@tcp:250:kworker.0 02000000:00000001:0.0:1786177011.917532:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177011.917536:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177011.917539:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786177011.917541:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177011.917543:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.917545:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1786177011.917548:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000100:00000001:1.0:1786177011.917549:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:0.0:1786177011.917554:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000040:1.0:1786177011.917554:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09315c00 x1872930235564160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786177011.917561:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177011.917565:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.917566:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1786177011.917570:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177011.917571:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1786177011.917575:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:1.0:1786177011.917575:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:0.0:1786177011.917580:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177011.917581:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177011.917585:0:2392:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000010:0.0:1786177011.917588:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000020:00000040:1.0:1786177011.917589:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000001:0.0:1786177011.917594:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177011.917594:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177011.917597:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:1.0:1786177011.917603:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00100000:0.0:1786177011.917604:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177011.917607:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177011.917611:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09315c00. 00000100:00000001:1.0:1786177011.917617:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177011.917621:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177011.917621:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.917626:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177011.917627:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177011.917634:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.917636:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177011.917646:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.917649:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177011.917650:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177011.917656:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1786177011.917657:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177011.917660:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177011.917662:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177011.917664:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1786177011.917665:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177011.917670:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:1.0:1786177011.917674:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1786177011.917676:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11686 00000400:00000200:0.0:1786177011.917684:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177011.917691:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177011.917695:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1786177011.917697:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177011.917699:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177011.917704:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177011.917704:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177011.917708:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177011.917709:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177011.917714:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177011.917716:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177011.917718:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177011.917723:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00100000:1.0:1786177011.917724:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564288:192.168.202.132@tcp:38:kworker.0 00000020:00000040:0.0:1786177011.917725:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786177011.917729:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177011.917732:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1786177011.917733:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 02000000:00000001:1.0:1786177011.917737:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1786177011.917738:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 02000000:00000001:1.0:1786177011.917741:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786177011.917743:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 02000000:00000001:1.0:1786177011.917745:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1786177011.917751:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000010:1.0:1786177011.917751:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 00000100:00000001:0.0:1786177011.917756:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177011.917757:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177011.917762:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177011.917767:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177011.917768:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:1.0:1786177011.917773:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000020:00000040:0.0:1786177011.917774:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000400:00000010:1.0:1786177011.917778:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00080000:0.0:1786177011.917780:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000200:1.0:1786177011.917782:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235564288, portal 10 00000100:00000001:0.0:1786177011.917784:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.917785:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1786177011.917789:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177011.917789:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794212352 : -110371915339264 : ffff9b9e09316a00) 02000000:00000001:0.0:1786177011.917794:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1786177011.917798:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786177011.917800:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177011.917804:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177011.917808:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177011.917814:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:0.0:1786177011.917815:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177011.917820:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177011.917821:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000010:0.0:1786177011.917827:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000400:00000010:1.0:1786177011.917828:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000020:00000040:0.0:1786177011.917833:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000200:1.0:1786177011.917834:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235564288, offset 0 02000000:00000001:0.0:1786177011.917838:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177011.917841:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177011.917844:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:0.0:1786177011.917845:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177011.917849:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177011.917851:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177011.917856:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:0.0:1786177011.917859:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000400:00000200:1.0:1786177011.917864:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:0.0:1786177011.917865:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177011.917871:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:0.0:1786177011.917875:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177011.917878:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177011.917885:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177011.917890:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:0.0:1786177011.917894:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177011.917898:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426475:426475:256:4294967295] 192.168.202.132@tcp LPNI seq info [426437:426437:8:4294967295] 00000100:00000001:0.0:1786177011.917899:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177011.917908:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177011.917910:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177011.917925:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177011.917926:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000040:0.0:1786177011.917933:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11696) 00000800:00000200:1.0:1786177011.917937:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177011.917945:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177011.917950:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177011.917954:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786177011.917959:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177011.917962:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1786177011.917967:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177011.917969:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177011.917972:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177011.917976:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177011.917980:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177011.917982:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177011.917985:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177011.917989:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.917998:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177011.918002:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918009:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.918010:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177011.918014:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177011.918018:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177011.918022:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177011.918024:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306358788 00000100:00000001:1.0:1786177011.918027:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306362624 00000100:00000001:1.0:1786177011.918029:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306358788 00000100:00000001:1.0:1786177011.918032:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306362624 00000100:00000001:1.0:1786177011.918034:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177011.918036:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.918038:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177011.918043:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177011/real 1786177011] req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177011.918057:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177011.918059:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177011.918061:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177011.918064:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177011.918066:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177011.918068:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177011.918071:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177011.918080:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918088:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177011.918089:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918091:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177011.918093:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177011.918096:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918104:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177011.918108:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918116:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177011.918119:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918121:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177011.918139:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918141:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177011.918143:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177011.918145:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177011.918148:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177011.918149:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177011.918153:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177011.918159:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564288:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177011.918161:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177011.918164:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918168:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177011.918169:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177011.918171:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177011.918172:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177011.918174:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177011.918176:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564416:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177011.918178:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177011.918179:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177011.918180:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177011.918182:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177011.918185:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177011.918188:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177011.918192:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177011.918194:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177011.918198:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177011.918201:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235564416, portal 10 00000100:00000001:1.0:1786177011.918203:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177011.918204:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215936 : -110371915335680 : ffff9b9e09317800) 00000100:00000040:1.0:1786177011.918210:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918217:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177011.918220:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177011.918222:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177011.918225:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235564416, offset 0 00000400:00000200:1.0:1786177011.918230:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177011.918234:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177011.918238:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177011.918242:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177011.918245:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177011.918247:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177011.918250:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177011.918253:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426476:426476:256:4294967295] 192.168.202.132@tcp LPNI seq info [426438:426438:8:4294967295] 00000400:00000200:1.0:1786177011.918258:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177011.918264:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177011.918269:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177011.918272:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177011.918277:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177011.918279:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786177011.918284:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177011.918287:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1786177011.918291:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177011.918293:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177011.918297:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177011.918301:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177011.918305:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177011.918307:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177011.918310:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177011.918314:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918322:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177011.918325:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918332:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.918334:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177011.918337:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177011.918340:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177011.918344:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177011.918347:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306358788 00000100:00000001:1.0:1786177011.918350:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306362624 00000100:00000001:1.0:1786177011.918352:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306358788 00000100:00000001:1.0:1786177011.918354:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306362624 00000100:00000001:1.0:1786177011.918378:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177011.918381:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.918383:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177011.918388:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177011/real 1786177011] req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177011.918396:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177011.918397:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177011.918399:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177011.918402:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177011.918404:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177011.918406:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177011.918410:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177011.918419:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918426:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177011.918427:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918430:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177011.918431:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177011.918436:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918443:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177011.918447:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918453:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177011.918456:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918457:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177011.918459:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918461:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177011.918463:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177011.918466:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177011.918469:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177011.918471:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177011.918475:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177011.918484:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564416:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177011.918488:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177011.918492:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918499:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177011.918500:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177011.918502:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177011.918503:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177011.918505:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177011.918507:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564544:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177011.918509:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177011.918510:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177011.918512:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177011.918513:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177011.918516:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177011.918517:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177011.918520:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177011.918522:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177011.918524:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177011.918526:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235564544, portal 4 00000100:00000001:1.0:1786177011.918528:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177011.918529:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794210560 : -110371915341056 : ffff9b9e09316300) 00000100:00000040:1.0:1786177011.918533:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918538:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177011.918540:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177011.918541:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177011.918543:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235564544, offset 0 00000400:00000200:1.0:1786177011.918545:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177011.918548:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177011.918550:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177011.918553:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177011.918555:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177011.918556:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177011.918558:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177011.918560:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426477:426477:256:4294967295] 192.168.202.132@tcp LPNI seq info [426439:426439:8:4294967295] 00000400:00000200:1.0:1786177011.918563:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177011.918569:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177011.918573:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177011.918576:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177011.918581:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177011.918584:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786177011.918588:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177011.918591:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1786177011.918595:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177011.918597:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177011.918599:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177011.918603:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177011.918606:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177011.918608:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177011.918610:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177011.918615:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918630:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177011.918634:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918640:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.918642:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177011.918645:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177011.918649:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177011.918652:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177011.918654:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306358789 00000100:00000001:1.0:1786177011.918656:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306362624 00000100:00000001:1.0:1786177011.918658:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306358789 00000100:00000001:1.0:1786177011.918661:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306362624 00000100:00000001:1.0:1786177011.918663:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177011.918665:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.918668:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177011.918672:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177011/real 1786177011] req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177011.918680:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177011.918682:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177011.918684:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177011.918686:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177011.918688:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177011.918690:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177011.918694:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177011.918702:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918708:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177011.918709:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918712:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177011.918713:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177011.918717:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918724:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177011.918728:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918735:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177011.918738:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918740:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177011.918741:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918743:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177011.918745:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177011.918747:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177011.918751:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177011.918753:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177011.918758:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177011.918766:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564544:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177011.918770:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918772:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177011.918775:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316a00 x1872930235564288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918780:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177011.918781:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177011.918783:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177011.918785:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177011.918786:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177011.918788:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177011.918790:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177011.918791:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:1.0:1786177011.918794:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177011.918795:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177011.918797:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316a00. 00000100:00000001:1.0:1786177011.918800:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177011.918802:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918804:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177011.918807:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317800 x1872930235564416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918814:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177011.918815:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177011.918816:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177011.918819:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177011.918820:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177011.918821:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177011.918824:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177011.918825:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:1.0:1786177011.918828:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177011.918829:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177011.918831:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317800. 00000100:00000001:1.0:1786177011.918833:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177011.918834:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918836:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177011.918840:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316300 x1872930235564544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918846:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177011.918848:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177011.918849:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177011.918851:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177011.918852:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177011.918853:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177011.918855:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177011.918857:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:1.0:1786177011.918859:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177011.918860:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177011.918862:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316300. 00000100:00000001:1.0:1786177011.918864:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177011.918865:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918867:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.918873:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177011.918875:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.918877:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177011.918879:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177011.918880:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177011.918884:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918892:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177011.918893:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177011.918895:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177011.918897:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177011.918899:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177011.918902:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564672:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177011.918905:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177011.918907:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177011.918909:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177011.918911:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177011.918914:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177011.918916:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177011.918919:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177011.918922:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177011.918926:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177011.918929:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235564672, portal 4 00000100:00000001:1.0:1786177011.918931:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177011.918933:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794202496 : -110371915349120 : ffff9b9e09314380) 00000100:00000040:1.0:1786177011.918939:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.918946:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177011.918948:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177011.918951:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177011.918953:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235564672, offset 0 00000400:00000200:1.0:1786177011.918958:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177011.918962:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177011.918966:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177011.918969:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177011.918972:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177011.918974:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177011.918977:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177011.918980:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426478:426478:256:4294967295] 192.168.202.132@tcp LPNI seq info [426440:426440:8:4294967295] 00000400:00000200:1.0:1786177011.918984:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177011.918990:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177011.918995:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177011.918999:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177011.919003:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177011.919006:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786177011.919010:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177011.919013:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1786177011.919017:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177011.919019:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177011.919022:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177011.919026:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177011.919029:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177011.919031:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177011.919033:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177011.919037:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.919045:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177011.919048:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177036 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.919055:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.919057:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177011.919061:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177011.919064:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177011.919067:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177011.919069:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306358791 00000100:00000001:1.0:1786177011.919072:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306362624 00000100:00000001:1.0:1786177011.919074:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306358791 00000100:00000001:1.0:1786177011.919077:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306362624 00000100:00000001:1.0:1786177011.919079:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177011.919081:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.919083:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177011.919088:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177011/real 1786177011] req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177011.919096:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177011.919098:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177011.919100:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177011.919103:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177011.919104:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177011.919106:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177011.919110:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177011.919117:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.919135:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177011.919136:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.919137:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177011.919138:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177011.919141:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.919146:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177011.919148:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.919154:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177011.919155:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.919156:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177011.919157:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.919158:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177011.919159:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177011.919161:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177011.919163:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177011.919164:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177011.919166:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177011.919171:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564672:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177011.919174:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.919175:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177011.919177:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314380 x1872930235564672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177036 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177011.919181:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177011.919182:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177011.919183:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177011.919185:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177011.919186:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177011.919187:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177011.919188:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177011.919189:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:1.0:1786177011.919192:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177011.919193:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177011.919195:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314380. 00000100:00000001:1.0:1786177011.919199:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177011.919200:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.919202:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177011.919206:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177011.919207:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177011.919209:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177011.919211:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177012.428435:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177012.428448:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177012.428452:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177012.428460:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177012.428466:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177012.428471:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177012.428474:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177012.428478:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177012.428483:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177012.812493:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177012.812503:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177012.812504:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177012.812508:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177012.812514:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177012.812516:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177012.812522:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177012.812528:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177012.812529:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177012.812531:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177012.812535:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177012.812537:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177012.812540:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177012.812540:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177012.812558:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177012.812635:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177012.812639:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177012.812643:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177013.452419:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177013.452429:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177013.452431:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177013.452438:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177013.452442:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177013.452445:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177013.452448:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177013.452450:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177013.452454:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177013.836500:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177013.836501:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177013.836508:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177013.836509:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177013.836514:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177013.836515:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177013.836518:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177013.836519:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177013.836521:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177013.836524:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177013.836525:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177013.836530:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177013.836533:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177013.836535:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177013.836538:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177013.836590:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177013.836594:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177013.836597:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177014.476438:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177014.476452:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177014.476456:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177014.476464:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177014.476470:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177014.476476:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177014.476481:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177014.476486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177014.476490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177014.860465:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177014.860471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177014.860474:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177014.860477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177014.860478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177014.860485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177014.860486:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177014.860489:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177014.860492:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177014.860495:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177014.860498:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177014.860499:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177014.860504:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177014.860507:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177014.860510:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177014.860565:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177014.860570:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177014.860574:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177015.500426:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177015.500434:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177015.500436:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177015.500441:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177015.500445:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177015.500448:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177015.500451:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177015.500453:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177015.500456:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177015.820426:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306362624, at 4306362688 00000100:00000040:1.0:1786177015.820436:0:206554:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306362624, at 4306362688 00000100:00000010:0.0:1786177015.820439:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ea0. 00000100:00000010:1.0:1786177015.820446:0:206554:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62600. 00000100:00000001:1.0:1786177015.884467:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177015.884470:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177015.884474:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177015.884476:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177015.884477:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177015.884483:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177015.884486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177015.884487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177015.884490:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177015.884491:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177015.884496:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177015.884498:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177015.884499:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177015.884500:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177015.884503:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177015.884561:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177015.884566:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177015.884570:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177016.524502:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177016.524513:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177016.524517:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177016.524525:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177016.524530:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177016.524534:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177016.524538:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177016.524542:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177016.524546:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177016.908450:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177016.908463:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177016.908465:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177016.908471:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177016.908475:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177016.908481:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177016.908485:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177016.908488:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177016.908492:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177016.908511:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177016.908520:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177016.908528:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177016.908530:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177016.908534:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177016.908535:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177016.908576:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177016.908581:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177016.908585:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177017.036522:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177017.036538:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177017.036541:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177017.036546:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177017.036551:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177017.036569:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11691 00000400:00000200:2.0:1786177017.036580:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177017.036589:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177017.036593:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177017.036597:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.036602:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177017.036605:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.036609:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177017.036614:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177017.036617:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177017.036620:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177017.036625:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177017.036628:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177017.036631:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177017.036634:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.036636:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177017.036639:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.036641:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177017.036644:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177017.036648:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177017.036651:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177017.036655:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177017.036659:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177017.036663:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177017.036666:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177017.036669:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177017.036683:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786177017.036689:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177017.036694:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177017.036696:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177017.036700:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177017.036704:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177017.036709:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177017.036719:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786177017.036727:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177017.036740:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177017.036752:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177017.036759:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177017.036764:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177017.036792:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177017.036796:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177017.036800:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177017.036802:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177017.036803:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177017.036807:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177017.036810:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11691 00000400:00000200:2.0:1786177017.036814:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177017.036817:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177017.036820:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177017.036824:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.036827:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177017.036829:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.036832:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177017.036835:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177017.036837:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177017.036839:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177017.036841:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1786177017.036843:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177017.036848:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177017.036849:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1786177017.036852:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177017.036854:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177017.036858:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.036861:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177017.036865:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.036869:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:1.0:1786177017.036872:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177017.036873:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177017.036880:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177017.036887:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177017.036892:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177017.036896:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1786177017.036901:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177017.036904:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177017.036906:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177017.036909:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1786177017.036913:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1786177017.036915:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1786177017.036920:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177017.036921:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177017.036926:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177017.036927:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177017.036934:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 00000100:00100000:1.0:1786177017.036935:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235564800:192.168.202.132@tcp:250:kworker.0 00000020:00000040:2.0:1786177017.036941:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000100:00000001:1.0:1786177017.036943:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177017.036946:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177017.036950:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177017.036950:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177017.036954:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177017.036955:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177017.036960:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1786177017.036960:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177017.036963:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177017.036969:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000010:2.0:1786177017.036970:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab400. 02000000:00000001:1.0:1786177017.036973:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177017.036978:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177017.036983:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:2.0:1786177017.036988:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177017.036990:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00000001:2.0:1786177017.037001:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1786177017.037002:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000001:2.0:1786177017.037006:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177017.037012:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177017.037014:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235564800, portal 25 00000100:00000001:1.0:1786177017.037020:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177017.037024:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620992 : -110371239930624 : ffff9b9e31735500) 00000100:00000040:1.0:1786177017.037037:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177017.037039:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177017.037045:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177017.037053:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177017.037057:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177017.037059:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1786177017.037060:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177017.037064:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:1.0:1786177017.037065:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00080000:2.0:1786177017.037070:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11691 00000400:00000010:1.0:1786177017.037085:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177017.037093:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235564800, offset 0 00000400:00000200:2.0:1786177017.037094:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177017.037100:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177017.037104:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:2.0:1786177017.037106:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177017.037112:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.037114:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177017.037117:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177017.037120:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.037124:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177017.037126:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000001:2.0:1786177017.037129:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177017.037132:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000400:00000200:1.0:1786177017.037134:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1786177017.037135:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1786177017.037140:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000040:2.0:1786177017.037140:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177017.037145:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177017.037148:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1786177017.037152:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177017.037153:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177017.037161:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426479:426479:256:4294967295] 192.168.202.132@tcp LPNI seq info [426441:426441:8:4294967295] 00000100:00000040:2.0:1786177017.037162:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.037169:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177017.037170:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177017.037174:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.037177:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177017.037180:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177017.037187:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177017.037192:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177017.037194:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 02000000:00000001:2.0:1786177017.037197:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177017.037202:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177017.037206:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177017.037208:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177017.037209:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177017.037214:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000800:00000010:1.0:1786177017.037218:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000008:00000001:2.0:1786177017.037218:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177017.037223:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177017.037229:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000800:00000200:1.0:1786177017.037233:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:2.0:1786177017.037234:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177017.037241:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.0:1786177017.037243:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 02000000:00000001:2.0:1786177017.037243:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177017.037247:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177017.037250:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177017.037253:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786177017.037254:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177017.037260:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 02000000:00000010:2.0:1786177017.037261:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000800:00000100:1.0:1786177017.037267:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177017.037268:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177017.037272:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000100:00100000:2.0:1786177017.037278:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177017.037280:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177017.037291:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1786177017.037292:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177017.037297:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177017.037300:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000100:00000040:2.0:1786177017.037304:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1786177017.037305:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177017.037312:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1786177017.037318:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177017.037321:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177017.037322:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177017.037331:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177017.037335:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177017.037338:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177017.037345:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177017.037351:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11691 00000100:00000001:1.0:1786177017.037353:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177017.037386:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.037393:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.037395:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1786177017.037399:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177017.037402:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177017.037408:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177017.037411:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:2.0:1786177017.037415:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177017.037416:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306367745 00000100:00000010:1.0:1786177017.037423:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62600. 00000100:00000040:2.0:1786177017.037423:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.037429:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177017.037434:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000040:2.0:1786177017.037434:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177017.037440:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177017.037444:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177017.037446:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306367745 00000020:00000040:2.0:1786177017.037447:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177017.037451:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000010:1.0:1786177017.037453:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62000. 00000020:00000040:2.0:1786177017.037456:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:1.0:1786177017.037457:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1786177017.037463:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177017.037464:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177017.037470:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177017.037470:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177017.037475:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1786177017.037479:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00100000:1.0:1786177017.037482:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177017/real 1786177017] req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177017.037484:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177017.037491:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000001:1.0:1786177017.037493:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1786177017.037495:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000001:1.0:1786177017.037496:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177017.037498:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000020:00000040:2.0:1786177017.037502:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177017.037506:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:1.0:1786177017.037510:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000100:00000001:2.0:1786177017.037511:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177017.037514:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 02000000:00000001:2.0:1786177017.037516:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177017.037518:0:2392:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1786177017.037522:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1786177017.037523:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177017.037529:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177017.037535:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:1.0:1786177017.037537:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1786177017.037539:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177017.037546:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.037548:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177017.037550:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.037552:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.037555:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000010:2.0:1786177017.037556:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000100:00000001:1.0:1786177017.037557:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1786177017.037561:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177017.037567:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1786177017.037570:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177017.037570:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177017.037575:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177017.037579:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177017.037583:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1786177017.037583:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177017.037588:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1786177017.037592:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:1.0:1786177017.037595:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177017.037599:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177017.037600:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.037601:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177017.037603:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.037606:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177017.037608:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:2.0:1786177017.037612:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1786177017.037613:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177017.037620:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177017.037624:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177017.037632:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177017.037632:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177017.037637:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177017.037645:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235564800:192.168.202.132@tcp:250:kworker.0 00000100:00000040:2.0:1786177017.037648:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.037650:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.037655:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177017.037662:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735500 x1872930235564800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177017.037665:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177017.037671:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177017.037672:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177017.037675:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:2.0:1786177017.037677:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000010:1.0:1786177017.037679:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 00000100:00000001:2.0:1786177017.037681:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1786177017.037683:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:2.0:1786177017.037685:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:1.0:1786177017.037686:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177017.037691:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000001:2.0:1786177017.037692:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000020:00000001:1.0:1786177017.037693:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177017.037697:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9400. 00000100:00080000:2.0:1786177017.037698:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11691 02000000:00000001:1.0:1786177017.037704:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:2.0:1786177017.037707:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177017.037708:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177017.037711:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735500. 00000400:00000200:2.0:1786177017.037714:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177017.037718:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177017.037721:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177017.037724:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177017.037725:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177017.037727:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177017.037731:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177017.037735:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177017.037738:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177017.037738:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177017.037741:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177017.037743:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177017.037745:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177017.037747:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1786177017.037747:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000001:1.0:1786177017.037750:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1786177017.037752:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1786177017.037756:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177017.037758:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177017.037765:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177017.037769:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1786177017.037770:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177017.037771:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177017.037775:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177017.037777:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1786177017.037780:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177017.037781:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177017.037788:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564928:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177017.037788:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177017.037791:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177017.037793:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1786177017.037795:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177017.037796:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177017.037799:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1786177017.037803:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000010:1.0:1786177017.037804:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177017.037807:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177017.037810:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000400:00000200:1.0:1786177017.037813:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177017.037818:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00080000:2.0:1786177017.037818:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000010:1.0:1786177017.037822:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000001:2.0:1786177017.037824:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177017.037827:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235564928, portal 10 02000000:00000001:2.0:1786177017.037828:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177017.037830:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177017.037833:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469630848 : -110371239920768 : ffff9b9e31737b80) 02000000:00000001:2.0:1786177017.037834:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177017.037839:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177017.037840:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177017.037843:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177017.037847:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177017.037850:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177017.037854:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000008:00000001:2.0:1786177017.037854:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177017.037858:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000001:2.0:1786177017.037859:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177017.037861:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235564928, offset 0 00000400:00000200:1.0:1786177017.037867:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000010:2.0:1786177017.037867:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 00000020:00000040:2.0:1786177017.037873:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000400:00000200:1.0:1786177017.037874:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177017.037879:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1786177017.037880:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 02000000:00000001:2.0:1786177017.037884:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177017.037885:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177017.037889:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:2.0:1786177017.037890:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1786177017.037893:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1786177017.037894:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1786177017.037897:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 02000000:00000001:2.0:1786177017.037899:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177017.037901:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426480:426480:256:4294967295] 192.168.202.132@tcp LPNI seq info [426442:426442:8:4294967295] 00000400:00000200:1.0:1786177017.037906:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:2.0:1786177017.037907:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000400:00000200:1.0:1786177017.037914:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177017.037915:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177017.037922:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00100000:2.0:1786177017.037926:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1786177017.037927:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177017.037934:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177017.037939:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786177017.037947:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177017.037948:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1786177017.037949:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1786177017.037955:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1786177017.037956:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177017.037960:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177017.037963:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1786177017.037966:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177017.037968:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177017.037972:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177017.037975:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177017.037979:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177017.037984:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177017.037988:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.037992:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177017.037995:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177017.037997:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11701) 00000100:00000001:1.0:1786177017.038009:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.038010:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177017.038012:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177017.038016:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177017.038018:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177017.038020:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306363908 00000100:00000001:1.0:1786177017.038022:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306367745 00000100:00000001:1.0:1786177017.038025:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306363908 00000100:00000001:1.0:1786177017.038026:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306367745 00000100:00000001:1.0:1786177017.038028:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177017.038030:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.038031:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177017.038033:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177017/real 1786177017] req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177017.038038:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177017.038039:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177017.038040:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177017.038042:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177017.038043:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177017.038044:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177017.038046:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177017.038051:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038055:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177017.038056:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038057:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177017.038058:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177017.038060:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038066:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177017.038068:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038085:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177017.038087:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038088:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177017.038089:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038090:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177017.038092:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177017.038094:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177017.038097:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177017.038098:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177017.038102:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177017.038107:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235564928:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177017.038110:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177017.038112:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038116:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177017.038117:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177017.038118:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177017.038119:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177017.038121:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177017.038123:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565056:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177017.038125:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177017.038127:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177017.038129:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177017.038131:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177017.038134:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177017.038136:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177017.038140:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177017.038143:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177017.038147:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177017.038150:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235565056, portal 10 00000100:00000001:1.0:1786177017.038152:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177017.038154:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469616512 : -110371239935104 : ffff9b9e31734380) 00000100:00000040:1.0:1786177017.038161:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038168:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177017.038170:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177017.038173:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177017.038176:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235565056, offset 0 00000400:00000200:1.0:1786177017.038181:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177017.038185:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177017.038189:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177017.038192:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177017.038195:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177017.038197:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177017.038200:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177017.038203:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426481:426481:256:4294967295] 192.168.202.132@tcp LPNI seq info [426443:426443:8:4294967295] 00000400:00000200:1.0:1786177017.038208:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177017.038215:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177017.038220:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177017.038223:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177017.038227:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177017.038230:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786177017.038234:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177017.038237:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1786177017.038240:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177017.038242:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177017.038244:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177017.038247:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177017.038249:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177017.038250:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177017.038252:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177017.038255:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038260:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177017.038262:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038267:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.038268:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177017.038270:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177017.038273:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177017.038275:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177017.038276:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306363908 00000100:00000001:1.0:1786177017.038278:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306367745 00000100:00000001:1.0:1786177017.038280:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306363908 00000100:00000001:1.0:1786177017.038281:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306367745 00000100:00000001:1.0:1786177017.038283:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177017.038284:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.038285:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177017.038287:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177017/real 1786177017] req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177017.038292:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177017.038293:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177017.038295:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177017.038296:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177017.038297:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177017.038300:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177017.038303:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177017.038312:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038319:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177017.038321:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038323:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177017.038324:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177017.038328:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038336:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177017.038340:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038346:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177017.038349:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038350:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177017.038352:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038353:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177017.038356:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177017.038378:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177017.038381:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177017.038384:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177017.038389:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177017.038398:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565056:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177017.038402:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177017.038406:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038413:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177017.038415:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177017.038417:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177017.038419:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177017.038421:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177017.038424:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565184:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177017.038428:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177017.038430:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177017.038432:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177017.038434:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177017.038437:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177017.038440:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177017.038444:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177017.038446:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177017.038450:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177017.038453:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235565184, portal 4 00000100:00000001:1.0:1786177017.038455:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177017.038456:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469624576 : -110371239927040 : ffff9b9e31736300) 00000100:00000040:1.0:1786177017.038463:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038471:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177017.038473:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177017.038476:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177017.038478:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235565184, offset 0 00000400:00000200:1.0:1786177017.038482:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177017.038487:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177017.038490:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177017.038493:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177017.038496:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177017.038498:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177017.038501:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177017.038504:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426482:426482:256:4294967295] 192.168.202.132@tcp LPNI seq info [426444:426444:8:4294967295] 00000400:00000200:1.0:1786177017.038509:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177017.038515:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177017.038520:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177017.038524:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177017.038528:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177017.038531:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786177017.038536:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177017.038539:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1786177017.038543:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177017.038545:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177017.038548:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177017.038551:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177017.038555:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177017.038557:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177017.038559:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177017.038563:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038571:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177017.038575:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038582:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.038583:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177017.038586:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177017.038590:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177017.038593:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177017.038595:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306363909 00000100:00000001:1.0:1786177017.038598:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306367745 00000100:00000001:1.0:1786177017.038600:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306363909 00000100:00000001:1.0:1786177017.038603:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306367745 00000100:00000001:1.0:1786177017.038605:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177017.038608:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.038610:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177017.038615:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177017/real 1786177017] req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177017.038623:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177017.038625:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177017.038627:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177017.038629:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177017.038631:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177017.038633:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177017.038636:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177017.038645:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038652:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177017.038654:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038656:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177017.038658:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177017.038662:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038669:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177017.038673:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038680:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177017.038683:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038684:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177017.038686:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038688:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177017.038689:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177017.038692:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177017.038696:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177017.038699:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177017.038703:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177017.038712:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565184:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177017.038716:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038718:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177017.038721:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737b80 x1872930235564928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038728:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177017.038730:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177017.038732:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177017.038735:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177017.038736:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177017.038737:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177017.038740:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177017.038742:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:1.0:1786177017.038744:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177017.038746:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177017.038748:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737b80. 00000100:00000001:1.0:1786177017.038750:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177017.038752:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038754:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177017.038757:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734380 x1872930235565056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038764:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177017.038765:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177017.038767:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177017.038770:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177017.038771:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177017.038773:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177017.038775:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177017.038777:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:1.0:1786177017.038780:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177017.038781:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177017.038782:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734380. 00000100:00000001:1.0:1786177017.038785:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177017.038786:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038788:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177017.038791:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736300 x1872930235565184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038798:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177017.038799:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177017.038801:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177017.038803:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177017.038805:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177017.038806:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177017.038808:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177017.038810:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:1.0:1786177017.038812:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177017.038814:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177017.038815:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736300. 00000100:00000001:1.0:1786177017.038818:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177017.038819:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038822:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.038826:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177017.038827:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.038830:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177017.038831:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177017.038833:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177017.038837:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038845:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177017.038847:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177017.038849:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177017.038851:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177017.038853:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177017.038856:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565312:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177017.038860:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177017.038862:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177017.038864:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177017.038866:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177017.038869:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177017.038871:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177017.038875:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177017.038878:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177017.038882:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177017.038885:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235565312, portal 4 00000100:00000001:1.0:1786177017.038887:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177017.038889:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469615616 : -110371239936000 : ffff9b9e31734000) 00000100:00000040:1.0:1786177017.038895:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.038902:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177017.038905:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177017.038908:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177017.038910:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235565312, offset 0 00000400:00000200:1.0:1786177017.038915:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177017.038919:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177017.038923:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177017.038926:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177017.038929:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177017.038931:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177017.038934:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177017.038937:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426483:426483:256:4294967295] 192.168.202.132@tcp LPNI seq info [426445:426445:8:4294967295] 00000400:00000200:1.0:1786177017.038941:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177017.038948:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177017.038952:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177017.038956:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177017.038960:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177017.038964:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786177017.038968:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177017.038971:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1786177017.038976:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177017.038978:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177017.038981:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177017.038985:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177017.038988:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177017.038990:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177017.038992:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177017.038996:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.039004:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177017.039007:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.039014:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.039016:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177017.039019:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177017.039022:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177017.039025:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177017.039027:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306363911 00000100:00000001:1.0:1786177017.039030:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306367745 00000100:00000001:1.0:1786177017.039033:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306363911 00000100:00000001:1.0:1786177017.039035:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306367745 00000100:00000001:1.0:1786177017.039038:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177017.039041:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.039043:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177017.039047:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177017/real 1786177017] req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177017.039057:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177017.039058:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177017.039061:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177017.039063:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177017.039065:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177017.039068:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177017.039084:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177017.039094:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.039101:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177017.039102:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.039104:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177017.039106:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177017.039109:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.039116:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177017.039120:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.039128:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177017.039131:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.039132:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177017.039134:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.039136:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177017.039137:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177017.039140:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177017.039144:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177017.039147:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177017.039151:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177017.039160:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565312:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177017.039165:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.039168:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177017.039171:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734000 x1872930235565312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177017.039179:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177017.039180:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177017.039182:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177017.039184:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177017.039186:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177017.039187:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177017.039190:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177017.039192:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:1.0:1786177017.039194:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177017.039195:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177017.039197:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734000. 00000100:00000001:1.0:1786177017.039200:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177017.039201:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.039203:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.039206:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177017.039208:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.039210:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177017.039212:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177017.548515:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177017.548524:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177017.548526:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177017.548533:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177017.548537:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177017.548542:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177017.548544:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177017.548547:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177017.548550:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.932469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177017.932477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177017.932478:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177017.932482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177017.932483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177017.932493:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177017.932493:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177017.932496:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.932500:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177017.932503:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177017.932507:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177017.932507:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.932514:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177017.932519:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177017.932523:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177017.932605:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177017.932612:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177017.932615:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177018.572515:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177018.572523:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177018.572526:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177018.572533:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177018.572537:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177018.572542:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177018.572544:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177018.572547:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177018.572551:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177018.956424:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177018.956430:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177018.956435:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177018.956436:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177018.956436:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177018.956443:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177018.956446:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177018.956447:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177018.956450:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177018.956453:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177018.956458:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177018.956462:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177018.956465:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177018.956467:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177018.956470:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177018.956502:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177018.956507:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177018.956511:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177019.596665:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177019.596677:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177019.596680:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177019.596689:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177019.596695:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177019.596701:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177019.596705:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177019.596709:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177019.596713:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177019.980486:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177019.980489:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177019.980498:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177019.980500:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177019.980506:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177019.980509:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177019.980515:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177019.980516:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177019.980518:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177019.980523:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177019.980525:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177019.980528:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177019.980531:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177019.980533:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177019.980533:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177019.980544:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177019.980545:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177019.980553:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177019.980553:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177019.980555:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177019.980562:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177019.980612:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177019.980626:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177019.980630:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177020.620480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177020.620491:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177020.620495:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177020.620501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177020.620507:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177020.620513:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177020.620525:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177020.620529:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177020.620533:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177020.940459:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306367745, at 4306367808 00000100:00000040:1.0:1786177020.940459:0:206554:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306367745, at 4306367808 00000100:00000010:1.0:1786177020.940474:0:206554:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62000. 00000100:00000010:0.0:1786177020.940476:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62600. 00000100:00000001:1.0:1786177021.004421:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177021.004429:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177021.004431:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177021.004432:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177021.004435:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177021.004441:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177021.004441:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177021.004443:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177021.004448:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177021.004448:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177021.004451:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177021.004454:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177021.004458:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177021.004463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177021.004466:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177021.004502:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177021.004507:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177021.004510:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177021.644544:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177021.644555:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177021.644558:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177021.644566:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177021.644571:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177021.644575:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177021.644579:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177021.644582:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177021.644586:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177022.028438:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177022.028438:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177022.028445:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177022.028446:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177022.028450:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177022.028451:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177022.028456:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177022.028459:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177022.028462:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177022.028464:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177022.028467:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.028468:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177022.028472:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.028473:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177022.028478:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.028479:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.028485:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177022.028486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177022.028490:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177022.028494:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177022.028496:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177022.028498:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177022.028501:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177022.028502:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177022.028504:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177022.028505:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177022.028507:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177022.028567:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177022.028572:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177022.028575:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177022.156437:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177022.156459:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177022.156462:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177022.156466:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177022.156470:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177022.156478:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11696 00000400:00000200:2.0:1786177022.156487:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177022.156494:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177022.156498:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177022.156501:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156510:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177022.156513:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156516:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177022.156520:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177022.156522:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177022.156526:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177022.156529:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177022.156532:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177022.156534:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177022.156538:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156540:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177022.156542:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156544:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177022.156547:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177022.156550:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156554:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177022.156557:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177022.156560:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177022.156563:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.156565:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177022.156567:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177022.156576:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 00000020:00000040:2.0:1786177022.156579:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177022.156583:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177022.156585:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.156588:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177022.156591:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177022.156595:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177022.156603:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177022.156609:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177022.156618:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177022.156628:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177022.156633:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177022.156637:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177022.156659:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177022.156663:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177022.156667:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177022.156668:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177022.156670:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177022.156673:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177022.156675:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11696 00000400:00000200:2.0:1786177022.156679:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177022.156682:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177022.156685:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177022.156687:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156690:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177022.156692:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156694:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177022.156697:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177022.156698:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177022.156700:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177022.156702:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177022.156704:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177022.156707:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177022.156710:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156712:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177022.156715:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156717:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177022.156719:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177022.156722:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156724:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177022.156727:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177022.156731:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177022.156733:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.156736:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177022.156740:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177022.156743:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.156745:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177022.156749:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000020:00000040:2.0:1786177022.156752:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177022.156754:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177022.156755:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.156757:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177022.156759:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177022.156761:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177022.156765:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786177022.156768:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177022.156774:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177022.156781:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177022.156784:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.156788:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1786177022.156788:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177022.156800:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.156803:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1786177022.156803:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177022.156808:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177022.156810:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177022.156812:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177022.156814:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177022.156817:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1786177022.156820:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11696 00000400:00000200:2.0:1786177022.156825:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177022.156828:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177022.156831:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177022.156834:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156837:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177022.156839:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156841:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177022.156845:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177022.156847:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177022.156848:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177022.156850:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177022.156852:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177022.156854:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177022.156855:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177022.156861:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156863:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177022.156865:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156867:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177022.156872:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177022.156875:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177022.156878:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177022.156880:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177022.156883:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177022.156889:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.156894:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177022.156896:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177022.156898:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.156900:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177022.156903:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000020:00000040:2.0:1786177022.156909:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177022.156912:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177022.156913:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.156915:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177022.156917:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177022.156920:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177022.156924:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000001:2.0:1786177022.156928:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.156932:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1786177022.156937:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177022.156951:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.156952:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177022.156957:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177022.156959:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1786177022.156963:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177022.156964:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177022.156974:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177022.156977:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177022.156981:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1786177022.156985:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.156986:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177022.156989:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177022.156993:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177022.156995:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11696 00000400:00000200:2.0:1786177022.156999:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177022.157002:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177022.157006:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1786177022.157009:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235565440:192.168.202.132@tcp:250:kworker.0 00000100:00000040:2.0:1786177022.157014:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177022.157015:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1786177022.157019:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177022.157038:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.157041:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177022.157044:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177022.157045:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177022.157047:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177022.157049:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177022.157052:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177022.157054:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 02000000:00000001:1.0:1786177022.157056:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1786177022.157058:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177022.157060:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.157063:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177022.157064:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1786177022.157069:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000010:1.0:1786177022.157071:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 00000100:00000001:2.0:1786177022.157072:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177022.157075:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177022.157079:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177022.157081:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177022.157084:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177022.157086:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177022.157090:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1786177022.157093:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177022.157095:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.157099:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177022.157101:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177022.157106:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177022.157107:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00000001:2.0:1786177022.157108:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177022.157116:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000010:2.0:1786177022.157116:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000020:00000040:2.0:1786177022.157121:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177022.157123:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177022.157124:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.157126:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177022.157128:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177022.157132:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177022.157135:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235565440, portal 25 02000000:00000010:2.0:1786177022.157138:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:1.0:1786177022.157139:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177022.157143:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469626368 : -110371239925248 : ffff9b9e31736a00) 00000100:00000001:2.0:1786177022.157145:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177022.157162:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1786177022.157168:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177022.157175:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177022.157178:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177022.157184:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177022.157193:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177022.157196:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177022.157205:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177022.157209:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1786177022.157209:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177022.157212:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1786177022.157216:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177022.157218:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177022.157224:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11696 00000400:00000200:2.0:1786177022.157229:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177022.157230:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000400:00000200:2.0:1786177022.157234:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000200:1.0:1786177022.157236:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235565440, offset 0 00000100:00080000:2.0:1786177022.157240:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177022.157245:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000040:2.0:1786177022.157248:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.157252:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177022.157256:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177022.157257:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.157262:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177022.157265:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000001:2.0:1786177022.157267:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177022.157270:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177022.157272:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177022.157276:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177022.157278:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177022.157280:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177022.157284:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.157286:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177022.157288:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177022.157291:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177022.157299:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1786177022.157304:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177022.157308:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177022.157311:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177022.157313:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1786177022.157316:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 02000000:00000001:2.0:1786177022.157317:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177022.157322:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.157324:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177022.157326:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000200:1.0:1786177022.157330:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000008:00000001:2.0:1786177022.157331:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.157334:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177022.157338:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000020:00000040:2.0:1786177022.157340:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177022.157346:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177022.157348:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.157350:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1786177022.157351:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 02000000:00000001:2.0:1786177022.157353:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177022.157389:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177022.157395:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000001:2.0:1786177022.157399:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177022.157405:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177022.157413:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177022.157416:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426484:426484:256:4294967295] 192.168.202.132@tcp LPNI seq info [426446:426446:8:4294967295] 00000100:00000001:2.0:1786177022.157418:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177022.157425:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177022.157440:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177022.157446:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11706) 00000400:00000200:1.0:1786177022.157448:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177022.157461:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177022.157486:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177022.157493:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177022.157501:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177022.157504:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786177022.157520:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177022.157523:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1786177022.157526:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177022.157528:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177022.157545:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177022.157549:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177022.157557:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177022.157559:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177022.157569:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177022.157573:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.157591:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177022.157594:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.157599:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.157600:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177022.157606:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177022.157609:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177022.157626:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306372865 00000100:00000010:1.0:1786177022.157630:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62000. 00000100:00000001:1.0:1786177022.157658:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177022.157663:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306372865 00000100:00000010:1.0:1786177022.157677:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62960. 00000100:00000001:1.0:1786177022.157690:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177022.157698:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177022.157701:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.157703:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177022.157714:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177022/real 1786177022] req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177022.157721:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177022.157723:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177022.157732:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177022.157735:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177022.157751:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177022.157753:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177022.157756:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177022.157762:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.157767:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177022.157780:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.157809:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177022.157812:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177022.157844:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.157863:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177022.157869:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.157877:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177022.157929:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.157958:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177022.157960:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.157963:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177022.157965:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177022.157982:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177022.157987:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177022.157991:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177022.157996:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177022.158090:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235565440:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177022.158097:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158099:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177022.158103:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736a00 x1872930235565440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158110:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177022.158112:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177022.158114:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177022.158117:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177022.158118:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177022.158120:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177022.158122:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177022.158124:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:1.0:1786177022.158129:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177022.158131:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177022.158133:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736a00. 00000100:00000001:1.0:1786177022.158136:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177022.158137:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158140:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158147:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177022.158149:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.158151:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177022.158152:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177022.158153:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177022.158158:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158166:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177022.158167:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177022.158170:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177022.158172:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177022.158175:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177022.158178:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565568:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177022.158182:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177022.158184:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177022.158186:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177022.158188:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177022.158191:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177022.158192:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177022.158196:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177022.158199:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177022.158203:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177022.158207:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235565568, portal 10 00000100:00000001:1.0:1786177022.158208:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177022.158209:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469625472 : -110371239926144 : ffff9b9e31736680) 00000100:00000040:1.0:1786177022.158214:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158219:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177022.158221:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177022.158222:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177022.158225:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235565568, offset 0 00000400:00000200:1.0:1786177022.158229:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177022.158233:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177022.158236:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177022.158239:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177022.158241:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177022.158243:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177022.158245:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177022.158249:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426485:426485:256:4294967295] 192.168.202.132@tcp LPNI seq info [426447:426447:8:4294967295] 00000400:00000200:1.0:1786177022.158253:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177022.158257:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177022.158261:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177022.158264:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177022.158267:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177022.158270:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786177022.158275:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177022.158278:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1786177022.158282:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177022.158284:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177022.158287:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177022.158291:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177022.158294:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177022.158297:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177022.158299:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177022.158303:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158313:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177022.158316:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158323:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.158324:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177022.158328:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177022.158332:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177022.158335:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177022.158337:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306369028 00000100:00000001:1.0:1786177022.158340:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306372865 00000100:00000001:1.0:1786177022.158343:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306369028 00000100:00000001:1.0:1786177022.158345:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306372865 00000100:00000001:1.0:1786177022.158348:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177022.158350:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.158352:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177022.158375:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177022/real 1786177022] req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177022.158393:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177022.158394:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177022.158396:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177022.158399:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177022.158401:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177022.158404:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177022.158408:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177022.158418:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158426:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177022.158428:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158430:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177022.158431:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177022.158435:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158442:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177022.158457:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158462:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177022.158465:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158466:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177022.158468:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158469:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177022.158472:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177022.158475:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177022.158478:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177022.158480:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177022.158484:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177022.158489:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565568:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177022.158492:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177022.158494:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158498:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177022.158499:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177022.158501:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177022.158503:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177022.158505:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177022.158508:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565696:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177022.158512:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177022.158514:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177022.158515:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177022.158516:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177022.158519:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177022.158520:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177022.158523:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177022.158525:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177022.158527:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177022.158530:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235565696, portal 10 00000100:00000001:1.0:1786177022.158531:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177022.158533:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469623680 : -110371239927936 : ffff9b9e31735f80) 00000100:00000040:1.0:1786177022.158538:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158545:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177022.158548:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177022.158550:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177022.158553:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235565696, offset 0 00000400:00000200:1.0:1786177022.158557:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177022.158562:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177022.158566:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177022.158569:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177022.158572:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177022.158574:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177022.158576:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177022.158578:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426486:426486:256:4294967295] 192.168.202.132@tcp LPNI seq info [426448:426448:8:4294967295] 00000400:00000200:1.0:1786177022.158581:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177022.158586:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177022.158589:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177022.158591:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177022.158594:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177022.158596:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786177022.158599:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177022.158601:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1786177022.158603:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177022.158605:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177022.158607:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177022.158611:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177022.158614:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177022.158616:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177022.158619:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177022.158622:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158631:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177022.158634:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158642:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.158644:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177022.158647:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177022.158649:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177022.158652:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177022.158654:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306369029 00000100:00000001:1.0:1786177022.158656:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306372865 00000100:00000001:1.0:1786177022.158657:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306369029 00000100:00000001:1.0:1786177022.158659:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306372865 00000100:00000001:1.0:1786177022.158660:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177022.158662:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.158664:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177022.158668:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177022/real 1786177022] req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177022.158676:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177022.158678:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177022.158680:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177022.158682:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177022.158684:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177022.158686:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177022.158690:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177022.158700:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158708:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177022.158710:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158711:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177022.158712:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177022.158715:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158723:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177022.158728:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158734:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177022.158736:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158737:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177022.158738:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158739:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177022.158741:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177022.158743:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177022.158744:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177022.158746:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177022.158748:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177022.158757:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565696:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177022.158760:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177022.158763:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158768:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177022.158769:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177022.158770:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177022.158772:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177022.158773:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177022.158775:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565824:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177022.158777:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177022.158778:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177022.158779:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177022.158781:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177022.158785:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177022.158787:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177022.158791:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177022.158793:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177022.158796:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177022.158798:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235565824, portal 4 00000100:00000001:1.0:1786177022.158800:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177022.158801:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469628160 : -110371239923456 : ffff9b9e31737100) 00000100:00000040:1.0:1786177022.158805:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158810:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177022.158811:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177022.158813:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177022.158814:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235565824, offset 0 00000400:00000200:1.0:1786177022.158816:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177022.158819:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177022.158821:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177022.158823:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177022.158825:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177022.158827:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177022.158828:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177022.158830:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426487:426487:256:4294967295] 192.168.202.132@tcp LPNI seq info [426449:426449:8:4294967295] 00000400:00000200:1.0:1786177022.158833:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177022.158837:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177022.158840:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177022.158842:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177022.158846:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177022.158849:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786177022.158853:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177022.158855:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1786177022.158860:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177022.158862:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177022.158865:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177022.158868:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177022.158872:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177022.158873:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177022.158877:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177022.158879:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158889:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177022.158891:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158896:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.158897:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177022.158899:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177022.158901:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177022.158903:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177022.158905:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306369029 00000100:00000001:1.0:1786177022.158906:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306372865 00000100:00000001:1.0:1786177022.158908:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306369029 00000100:00000001:1.0:1786177022.158909:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306372865 00000100:00000001:1.0:1786177022.158911:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177022.158912:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.158913:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177022.158915:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177022/real 1786177022] req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177022.158921:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177022.158921:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177022.158923:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177022.158924:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177022.158925:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177022.158927:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177022.158929:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177022.158934:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158938:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177022.158939:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158940:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177022.158941:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177022.158943:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158948:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177022.158950:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158954:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177022.158955:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158956:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177022.158957:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.158958:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177022.158959:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177022.158961:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177022.158963:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177022.158964:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177022.158966:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177022.158971:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565824:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177022.158973:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177022.158975:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.158979:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177022.158980:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177022.158982:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177022.158983:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177022.158984:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177022.158987:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565952:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177022.158989:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177022.158990:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177022.158991:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177022.158993:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177022.158995:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177022.158996:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177022.158998:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177022.158999:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177022.159002:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177022.159004:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235565952, portal 4 00000100:00000001:1.0:1786177022.159005:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177022.159006:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469618304 : -110371239933312 : ffff9b9e31734a80) 00000100:00000040:1.0:1786177022.159010:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.159014:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177022.159015:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177022.159016:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177022.159018:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235565952, offset 0 00000400:00000200:1.0:1786177022.159020:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177022.159038:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177022.159042:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177022.159044:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177022.159045:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177022.159047:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177022.159049:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177022.159051:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426488:426488:256:4294967295] 192.168.202.132@tcp LPNI seq info [426450:426450:8:4294967295] 00000400:00000200:1.0:1786177022.159054:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177022.159058:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177022.159061:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177022.159063:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177022.159066:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177022.159068:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786177022.159071:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177022.159072:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1786177022.159075:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177022.159076:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177022.159078:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177022.159081:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177022.159082:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177022.159084:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177022.159085:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177022.159088:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.159093:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177022.159095:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177047 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.159099:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.159101:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177022.159102:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177022.159104:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177022.159106:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177022.159108:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306369031 00000100:00000001:1.0:1786177022.159110:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306372865 00000100:00000001:1.0:1786177022.159111:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306369031 00000100:00000001:1.0:1786177022.159112:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306372865 00000100:00000001:1.0:1786177022.159114:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177022.159115:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.159116:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177022.159118:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177022/real 1786177022] req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177022.159123:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177022.159124:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177022.159125:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177022.159127:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177022.159128:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177022.159129:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177022.159131:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177022.159136:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.159140:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177022.159141:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.159142:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177022.159143:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177022.159145:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.159149:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177022.159151:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.159156:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177022.159157:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.159158:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177022.159159:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.159160:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177022.159161:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177022.159163:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177022.159164:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177022.159165:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177022.159168:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177022.159176:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235565952:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177022.159179:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.159180:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177022.159182:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736680 x1872930235565568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.159186:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177022.159187:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177022.159188:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177022.159190:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177022.159191:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177022.159191:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177022.159193:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177022.159194:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:1.0:1786177022.159195:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177022.159196:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177022.159197:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736680. 00000100:00000001:1.0:1786177022.159199:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177022.159200:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.159201:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177022.159203:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735f80 x1872930235565696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.159207:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177022.159208:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177022.159209:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177022.159210:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177022.159211:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177022.159211:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177022.159212:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177022.159213:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:1.0:1786177022.159214:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177022.159215:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177022.159216:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735f80. 00000100:00000001:1.0:1786177022.159218:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177022.159219:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.159220:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177022.159222:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737100 x1872930235565824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.159226:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177022.159227:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177022.159227:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177022.159229:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177022.159229:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177022.159230:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177022.159231:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177022.159232:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:1.0:1786177022.159233:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177022.159234:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177022.159234:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737100. 00000100:00000001:1.0:1786177022.159236:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177022.159237:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.159237:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177022.159239:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734a80 x1872930235565952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177047 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177022.159243:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177022.159243:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177022.159244:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177022.159246:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177022.159246:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177022.159247:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177022.159248:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177022.159249:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:1.0:1786177022.159250:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177022.159251:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177022.159252:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734a80. 00000100:00000001:1.0:1786177022.159253:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177022.159254:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.159255:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177022.159257:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177022.159258:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177022.159260:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177022.159261:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177022.668691:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177022.668703:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177022.668706:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177022.668713:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177022.668717:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177022.668723:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177022.668728:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177022.668732:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177022.668736:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177023.052469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177023.052473:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177023.052477:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177023.052479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177023.052481:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177023.052488:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177023.052489:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177023.052492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177023.052495:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177023.052499:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177023.052500:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177023.052504:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177023.052504:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177023.052507:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177023.052510:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177023.052565:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177023.052570:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177023.052574:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177023.692506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177023.692534:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177023.692538:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177023.692551:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177023.692560:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177023.692568:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177023.692573:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177023.692578:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177023.692583:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177024.076430:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177024.076434:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177024.076437:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177024.076438:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177024.076441:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177024.076444:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177024.076446:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177024.076447:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177024.076451:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177024.076454:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177024.076455:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177024.076456:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177024.076459:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177024.076463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177024.076465:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177024.076525:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177024.076532:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177024.076534:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177024.716528:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177024.716541:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177024.716544:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177024.716553:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177024.716557:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177024.716562:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177024.716566:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177024.716569:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177024.716573:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177025.100439:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177025.100440:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177025.100445:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177025.100447:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177025.100451:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177025.100452:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177025.100454:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177025.100458:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177025.100459:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177025.100460:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177025.100466:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177025.100470:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177025.100474:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177025.100476:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177025.100479:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177025.100505:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177025.100509:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177025.100514:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177025.740589:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177025.740596:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177025.740597:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177025.740602:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177025.740604:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177025.740607:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177025.740609:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177025.740611:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177025.740613:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177026.060458:0:206554:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306372865, at 4306372928 00000100:00000040:0.0:1786177026.060461:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306372865, at 4306372928 00000100:00000010:1.0:1786177026.060473:0:206554:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62960. 00000100:00000010:0.0:1786177026.060475:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62000. 00000100:00000001:1.0:1786177026.124453:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177026.124456:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177026.124460:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177026.124460:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177026.124463:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177026.124467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177026.124468:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177026.124470:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177026.124472:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177026.124474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177026.124477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177026.124478:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177026.124481:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177026.124483:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177026.124485:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177026.124537:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177026.124541:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177026.124543:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177026.764488:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177026.764507:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177026.764511:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177026.764516:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177026.764522:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177026.764526:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177026.764531:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177026.764534:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177026.764538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.148477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177027.148484:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177027.148484:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177027.148487:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177027.148490:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177027.148492:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177027.148497:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177027.148498:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177027.148500:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177027.148501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.148506:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177027.148507:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177027.148509:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177027.148510:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.148513:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177027.148575:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177027.148579:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177027.148584:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177027.276490:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177027.276498:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177027.276501:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177027.276504:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177027.276508:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177027.276514:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11701 00000400:00000200:3.0:1786177027.276522:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177027.276526:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177027.276529:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177027.276531:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276534:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177027.276536:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276537:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177027.276541:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177027.276542:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177027.276545:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177027.276547:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177027.276549:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177027.276550:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177027.276553:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276555:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177027.276556:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276557:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177027.276560:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177027.276562:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276564:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177027.276566:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177027.276575:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177027.276578:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.276579:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177027.276581:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177027.276589:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786177027.276592:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177027.276595:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177027.276596:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.276599:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177027.276602:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177027.276605:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177027.276611:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786177027.276616:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177027.276624:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177027.276632:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177027.276635:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177027.276639:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177027.276656:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177027.276659:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177027.276666:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177027.276667:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177027.276668:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177027.276670:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177027.276673:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11701 00000400:00000200:3.0:1786177027.276675:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177027.276677:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177027.276679:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177027.276681:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276683:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177027.276684:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276686:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177027.276688:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177027.276689:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177027.276691:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177027.276692:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177027.276693:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177027.276695:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177027.276697:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276699:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177027.276700:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1786177027.276701:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177027.276705:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177027.276705:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177027.276707:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:1.0:1786177027.276709:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1786177027.276711:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276714:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177027.276717:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1786177027.276719:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177027.276725:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177027.276730:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.276735:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177027.276739:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1786177027.276740:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1786177027.276744:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1786177027.276744:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.276750:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.276752:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:3.0:1786177027.276753:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 02000000:00000001:1.0:1786177027.276758:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1786177027.276758:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177027.276765:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177027.276766:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177027.276767:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.276772:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177027.276774:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1786177027.276776:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235566080:192.168.202.132@tcp:250:kworker.0 02000000:00000001:3.0:1786177027.276777:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177027.276783:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:1.0:1786177027.276784:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1786177027.276787:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.276789:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177027.276792:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177027.276795:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177027.276796:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177027.276803:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177027.276807:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.276812:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177027.276815:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177027.276817:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177027.276820:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00000040:3.0:1786177027.276822:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1786177027.276827:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000001:3.0:1786177027.276830:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177027.276833:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1786177027.276835:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235566080, portal 25 00000100:00080000:3.0:1786177027.276836:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177027.276840:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1786177027.276841:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177027.276844:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177027.276845:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280820736 : -110371428730880 : ffff9b9e26327800) 00000100:00000001:3.0:1786177027.276853:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:1.0:1786177027.276854:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177027.276858:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11701 00000400:00000200:3.0:1786177027.276867:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177027.276872:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177027.276875:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177027.276878:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1786177027.276879:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276883:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177027.276887:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177027.276888:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276892:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:1.0:1786177027.276895:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177027.276899:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235566080, offset 0 00000020:00000001:3.0:1786177027.276903:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177027.276907:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000400:00000200:1.0:1786177027.276912:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000020:00000001:3.0:1786177027.276912:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177027.276917:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177027.276920:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177027.276922:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:3.0:1786177027.276926:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177027.276934:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000040:3.0:1786177027.276934:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276941:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177027.276944:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1786177027.276945:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276949:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177027.276951:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:1.0:1786177027.276954:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00080000:3.0:1786177027.276956:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276960:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177027.276961:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1786177027.276964:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1786177027.276966:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177027.276970:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177027.276971:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.276987:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1786177027.276990:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426489:426489:256:4294967295] 192.168.202.132@tcp LPNI seq info [426451:426451:8:4294967295] 00000008:00100000:3.0:1786177027.276992:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177027.276999:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000008:00000001:3.0:1786177027.276999:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.277004:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177027.277010:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000010:3.0:1786177027.277014:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000800:00000200:1.0:1786177027.277017:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177027.277019:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000800:00000010:1.0:1786177027.277023:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 02000000:00000001:3.0:1786177027.277024:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177027.277027:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.277030:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.0:1786177027.277033:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786177027.277033:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.0:1786177027.277038:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 02000000:00000001:3.0:1786177027.277039:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786177027.277047:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000010:3.0:1786177027.277049:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000800:00000200:1.1:1786177027.277051:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1786177027.277057:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177027.277057:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177027.277062:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000100:00100000:3.0:1786177027.277068:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177027.277070:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177027.277081:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1786177027.277087:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177027.277090:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000100:00000001:3.0:1786177027.277091:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177027.277095:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177027.277100:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1786177027.277106:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177027.277109:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177027.277129:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.277133:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1786177027.277134:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177027.277139:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000040:1.0:1786177027.277140:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177027.277141:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177027.277145:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177027.277153:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177027.277158:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177027.277160:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11701 00000100:00000001:1.0:1786177027.277162:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1786177027.277170:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177027.277172:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177027.277179:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177027.277182:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306377984 00000400:00000200:3.0:1786177027.277184:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000010:1.0:1786177027.277190:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62960. 00000100:00080000:3.0:1786177027.277190:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177027.277196:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.277199:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177027.277201:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1786177027.277201:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.277206:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177027.277208:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177027.277209:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:1.0:1786177027.277210:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306377984 00000020:00000001:3.0:1786177027.277211:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177027.277215:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000010:1.0:1786177027.277216:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62e40. 00000020:00000001:3.0:1786177027.277219:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177027.277220:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1786177027.277222:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177027.277224:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1786177027.277228:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177027.277229:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.277230:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177027.277233:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1786177027.277234:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.277237:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00100000:1.0:1786177027.277240:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177027/real 1786177027] req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177027.277240:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177027.277247:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177027.277250:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177027.277254:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177027.277258:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177027.277261:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1786177027.277263:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177027.277267:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1786177027.277268:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:1.0:1786177027.277269:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000008:00100000:3.0:1786177027.277272:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000200:1.0:1786177027.277275:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000008:00000001:3.0:1786177027.277276:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.277281:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177027.277282:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000010:3.0:1786177027.277285:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000100:00000001:1.0:1786177027.277287:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:3.0:1786177027.277288:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177027.277292:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177027.277294:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177027.277297:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177027.277297:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177027.277298:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177027.277301:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177027.277306:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177027.277311:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177027.277316:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786177027.277318:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177027.277348:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1786177027.277348:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177027.277352:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177027.277353:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.277356:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177027.277386:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177027.277391:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.277397:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177027.277402:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177027.277403:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.277426:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177027.277431:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177027.277431:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.277436:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1786177027.277437:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177027.277438:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.277443:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1786177027.277443:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177027.277445:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177027.277446:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177027.277450:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1786177027.277452:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1786177027.277454:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177027.277459:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00080000:3.0:1786177027.277459:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11701 00000100:00000001:1.0:1786177027.277467:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:3.0:1786177027.277468:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177027.277473:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177027.277476:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177027.277479:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177027.277480:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.277485:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177027.277489:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.277494:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177027.277499:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177027.277501:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177027.277504:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:1.0:1786177027.277505:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235566080:192.168.202.132@tcp:250:kworker.0 00000020:00000040:3.0:1786177027.277506:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177027.277511:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177027.277514:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177027.277515:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177027.277519:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177027.277520:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1786177027.277523:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177027.277527:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327800 x1872930235566080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177027.277529:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177027.277534:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177027.277539:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:1.0:1786177027.277541:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1786177027.277544:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177027.277546:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177027.277547:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177027.277549:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177027.277552:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.277554:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1786177027.277554:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000010:1.0:1786177027.277557:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 00000008:00100000:3.0:1786177027.277558:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 02000000:00000001:1.0:1786177027.277563:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00000001:3.0:1786177027.277563:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.277565:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177027.277567:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177027.277570:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000010:3.0:1786177027.277570:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000001:1.0:1786177027.277572:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000040:3.0:1786177027.277574:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000010:1.0:1786177027.277576:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 02000000:00000001:3.0:1786177027.277578:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177027.277580:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.277581:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1786177027.277582:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177027.277584:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1786177027.277585:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:1.0:1786177027.277587:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327800. 02000000:00000001:3.0:1786177027.277590:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.277591:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177027.277594:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.277596:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1786177027.277598:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:1.0:1786177027.277603:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177027.277603:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.277606:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.277609:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177027.277611:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1786177027.277613:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177027.277614:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177027.277620:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.277637:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177027.277640:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.277644:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177027.277647:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177027.277651:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177027.277652:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1786177027.277655:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566208:192.168.202.132@tcp:38:kworker.0 00000100:00000001:3.0:1786177027.277658:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.277659:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177027.277663:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177027.277665:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.277668:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1786177027.277669:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177027.277671:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177027.277675:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177027.277682:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177027.277686:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177027.277690:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000001:3.0:1786177027.277693:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177027.277695:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235566208, portal 10 00000100:00000001:1.0:1786177027.277698:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177027.277700:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818048 : -110371428733568 : ffff9b9e26326d80) 00000100:00000040:3.0:1786177027.277701:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11711) 00000100:00000040:1.0:1786177027.277708:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.277714:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177027.277716:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177027.277718:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177027.277721:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235566208, offset 0 00000400:00000200:1.0:1786177027.277726:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177027.277731:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177027.277735:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177027.277739:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177027.277741:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177027.277744:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177027.277747:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177027.277750:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426490:426490:256:4294967295] 192.168.202.132@tcp LPNI seq info [426452:426452:8:4294967295] 00000400:00000200:1.0:1786177027.277755:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177027.277763:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177027.277768:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177027.277772:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177027.277776:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177027.277778:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786177027.277781:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177027.277783:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1786177027.277786:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177027.277787:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177027.277789:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177027.277792:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177027.277794:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177027.277795:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177027.277797:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177027.277800:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.277806:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177027.277808:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.277812:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.277813:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177027.277817:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177027.277819:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177027.277822:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177027.277824:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306374148 00000100:00000001:1.0:1786177027.277826:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306377984 00000100:00000001:1.0:1786177027.277827:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306374148 00000100:00000001:1.0:1786177027.277829:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306377984 00000100:00000001:1.0:1786177027.277830:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177027.277831:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.277832:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177027.277835:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177027/real 1786177027] req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177027.277840:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177027.277841:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177027.277842:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177027.277844:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177027.277845:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177027.277846:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177027.277848:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177027.277854:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.277860:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177027.277862:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.277864:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177027.277865:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177027.277869:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.277876:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177027.277881:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.277889:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177027.277892:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.277893:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177027.277895:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.277897:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177027.277899:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177027.277902:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177027.277905:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177027.277908:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177027.277911:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177027.277917:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566208:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177027.277919:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177027.277922:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.277928:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177027.277930:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.277932:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177027.277934:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177027.277936:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177027.277940:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566336:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177027.277943:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177027.277945:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177027.277947:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.277949:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177027.277953:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177027.277955:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177027.277959:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177027.277962:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177027.277966:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177027.277969:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235566336, portal 10 00000100:00000001:1.0:1786177027.277971:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177027.277985:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280810880 : -110371428740736 : ffff9b9e26325180) 00000100:00000040:1.0:1786177027.277991:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.277996:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177027.277998:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177027.278000:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177027.278002:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235566336, offset 0 00000400:00000200:1.0:1786177027.278004:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177027.278008:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177027.278011:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177027.278013:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177027.278015:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177027.278016:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177027.278018:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177027.278020:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426491:426491:256:4294967295] 192.168.202.132@tcp LPNI seq info [426453:426453:8:4294967295] 00000400:00000200:1.0:1786177027.278023:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177027.278027:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177027.278030:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177027.278032:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177027.278034:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177027.278036:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786177027.278039:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177027.278041:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1786177027.278044:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177027.278045:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177027.278047:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177027.278050:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177027.278052:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177027.278053:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177027.278055:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177027.278057:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278063:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177027.278065:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278069:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.278070:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177027.278072:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177027.278074:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177027.278076:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177027.278077:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306374148 00000100:00000001:1.0:1786177027.278079:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306377984 00000100:00000001:1.0:1786177027.278081:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306374148 00000100:00000001:1.0:1786177027.278082:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306377984 00000100:00000001:1.0:1786177027.278084:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177027.278085:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.278086:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177027.278089:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177027/real 1786177027] req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177027.278093:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177027.278094:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177027.278095:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177027.278097:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177027.278098:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177027.278099:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177027.278101:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177027.278106:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278111:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177027.278112:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278113:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177027.278114:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177027.278117:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278121:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177027.278123:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278127:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177027.278128:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278129:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177027.278130:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278131:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177027.278133:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177027.278134:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177027.278136:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177027.278137:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177027.278140:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177027.278144:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566336:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177027.278148:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177027.278151:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278158:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177027.278160:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.278162:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177027.278164:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177027.278166:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177027.278170:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566464:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177027.278173:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177027.278175:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177027.278177:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.278179:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177027.278182:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177027.278184:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177027.278188:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177027.278191:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177027.278195:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177027.278198:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235566464, portal 4 00000100:00000001:1.0:1786177027.278200:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177027.278201:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280813568 : -110371428738048 : ffff9b9e26325c00) 00000100:00000040:1.0:1786177027.278207:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278211:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177027.278213:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177027.278215:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177027.278216:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235566464, offset 0 00000400:00000200:1.0:1786177027.278219:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177027.278222:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177027.278224:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177027.278226:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177027.278228:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177027.278230:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177027.278231:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177027.278234:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426492:426492:256:4294967295] 192.168.202.132@tcp LPNI seq info [426454:426454:8:4294967295] 00000400:00000200:1.0:1786177027.278236:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177027.278241:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177027.278245:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177027.278248:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177027.278252:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177027.278255:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786177027.278259:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177027.278262:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1786177027.278266:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177027.278268:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177027.278271:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177027.278275:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177027.278278:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177027.278280:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177027.278283:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177027.278287:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278296:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177027.278299:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278306:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.278308:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177027.278311:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177027.278314:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177027.278318:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177027.278320:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306374148 00000100:00000001:1.0:1786177027.278323:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306377984 00000100:00000001:1.0:1786177027.278325:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306374148 00000100:00000001:1.0:1786177027.278327:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306377984 00000100:00000001:1.0:1786177027.278330:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177027.278332:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.278334:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177027.278338:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177027/real 1786177027] req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177027.278345:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177027.278346:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177027.278348:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177027.278350:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177027.278351:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177027.278352:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177027.278354:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177027.278376:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278381:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177027.278382:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278383:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177027.278384:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177027.278387:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278391:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177027.278393:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278398:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177027.278399:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278400:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177027.278401:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278402:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177027.278403:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177027.278405:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177027.278407:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177027.278408:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177027.278411:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177027.278416:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566464:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177027.278419:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278420:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177027.278422:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326d80 x1872930235566208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278426:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177027.278427:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177027.278428:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177027.278430:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177027.278431:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177027.278432:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177027.278434:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177027.278435:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:1.0:1786177027.278437:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177027.278438:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177027.278439:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326d80. 00000100:00000001:1.0:1786177027.278441:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177027.278442:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278443:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177027.278445:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325180 x1872930235566336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278449:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177027.278450:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177027.278451:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177027.278453:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177027.278454:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177027.278455:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177027.278457:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177027.278458:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:1.0:1786177027.278460:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177027.278462:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177027.278463:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325180. 00000100:00000001:1.0:1786177027.278465:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177027.278467:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278468:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177027.278472:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325c00 x1872930235566464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278479:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177027.278480:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177027.278482:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177027.278484:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177027.278486:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177027.278487:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177027.278489:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177027.278492:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:1.0:1786177027.278494:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177027.278495:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177027.278497:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325c00. 00000100:00000001:1.0:1786177027.278499:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177027.278500:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278502:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278506:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177027.278507:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.278509:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177027.278510:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177027.278511:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177027.278514:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278519:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177027.278520:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.278522:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177027.278523:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177027.278524:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177027.278526:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566592:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177027.278528:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177027.278530:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177027.278531:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177027.278533:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177027.278535:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177027.278536:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177027.278539:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177027.278541:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177027.278543:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177027.278545:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235566592, portal 4 00000100:00000001:1.0:1786177027.278547:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177027.278547:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280815360 : -110371428736256 : ffff9b9e26326300) 00000100:00000040:1.0:1786177027.278552:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278556:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177027.278558:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177027.278559:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177027.278561:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235566592, offset 0 00000400:00000200:1.0:1786177027.278563:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177027.278566:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177027.278568:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177027.278570:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177027.278572:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177027.278574:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177027.278575:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177027.278577:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426493:426493:256:4294967295] 192.168.202.132@tcp LPNI seq info [426455:426455:8:4294967295] 00000400:00000200:1.0:1786177027.278580:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177027.278584:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177027.278587:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177027.278589:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177027.278592:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177027.278593:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786177027.278596:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177027.278598:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1786177027.278600:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177027.278602:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177027.278604:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177027.278606:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177027.278608:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177027.278609:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177027.278611:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177027.278613:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278619:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177027.278620:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177052 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278625:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.278626:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177027.278628:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177027.278630:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177027.278632:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177027.278633:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306374151 00000100:00000001:1.0:1786177027.278635:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306377984 00000100:00000001:1.0:1786177027.278636:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306374151 00000100:00000001:1.0:1786177027.278638:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306377984 00000100:00000001:1.0:1786177027.278639:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177027.278640:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.278641:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177027.278643:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177027/real 1786177027] req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177027.278648:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177027.278649:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177027.278650:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177027.278652:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177027.278653:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177027.278654:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177027.278657:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177027.278662:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278666:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177027.278666:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278668:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177027.278668:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177027.278671:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278675:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177027.278677:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278681:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177027.278682:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278683:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177027.278684:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278685:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177027.278686:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177027.278688:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177027.278690:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177027.278691:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177027.278694:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177027.278698:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566592:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177027.278701:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278702:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177027.278704:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326300 x1872930235566592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177052 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177027.278708:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177027.278709:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177027.278711:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177027.278712:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177027.278713:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177027.278713:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177027.278715:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177027.278716:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:1.0:1786177027.278717:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177027.278718:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177027.278719:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326300. 00000100:00000001:1.0:1786177027.278721:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177027.278722:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278724:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177027.278727:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177027.278729:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177027.278731:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177027.278732:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177027.788565:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177027.788573:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177027.788576:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177027.788582:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177027.788585:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177027.788589:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177027.788592:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177027.788595:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177027.788598:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177028.172473:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177028.172478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177028.172480:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177028.172482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177028.172485:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177028.172487:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177028.172491:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177028.172492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177028.172494:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177028.172496:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177028.172498:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177028.172502:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177028.172504:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177028.172505:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177028.172507:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177028.172562:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177028.172566:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177028.172570:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177028.812497:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177028.812505:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177028.812507:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177028.812513:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177028.812517:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177028.812521:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177028.812524:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177028.812534:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177028.812538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177029.196472:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177029.196478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177029.196481:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177029.196483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177029.196485:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177029.196490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177029.196492:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177029.196493:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177029.196498:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177029.196501:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177029.196504:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177029.196505:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177029.196512:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177029.196515:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177029.196519:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177029.196571:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177029.196576:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177029.196581:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177029.836482:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177029.836490:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177029.836493:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177029.836497:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177029.836500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177029.836504:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177029.836506:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177029.836508:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177029.836511:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177030.220450:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177030.220458:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177030.220461:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177030.220462:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177030.220467:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177030.220468:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177030.220473:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177030.220474:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177030.220479:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177030.220480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177030.220483:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177030.220485:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177030.220488:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177030.220489:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177030.220491:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177030.220494:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177030.220498:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177030.220501:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177030.220504:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177030.220508:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177030.220510:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177030.220559:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177030.220563:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177030.220566:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177030.860495:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177030.860507:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177030.860511:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177030.860519:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177030.860524:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177030.860529:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177030.860533:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177030.860537:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177030.860541:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177031.180468:0:206554:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306377984, at 4306378048 00000100:00000010:1.0:1786177031.180481:0:206554:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62e40. 00000100:00000040:0.0:1786177031.180511:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306377984, at 4306378048 00000100:00000010:0.0:1786177031.180526:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62960. 00000100:00000001:1.0:1786177031.244477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177031.244485:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177031.244487:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177031.244491:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177031.244494:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177031.244497:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177031.244499:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177031.244501:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177031.244504:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177031.244532:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177031.244540:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177031.244546:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177031.244548:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177031.244552:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177031.244553:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177031.244611:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177031.244615:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177031.244618:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177031.884497:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177031.884506:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177031.884508:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177031.884513:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177031.884517:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177031.884520:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177031.884524:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177031.884526:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177031.884529:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.268450:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177032.268457:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177032.268458:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177032.268461:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177032.268464:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:1.0:1786177032.268466:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177032.268468:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177032.268470:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177032.268473:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.268474:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.268479:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177032.268480:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177032.268482:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177032.268484:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.268485:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177032.268490:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177032.268494:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177032.268499:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177032.268500:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177032.268504:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177032.268506:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177032.268508:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177032.268509:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177032.268511:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177032.268512:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177032.268514:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177032.268516:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177032.268561:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177032.268567:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177032.268571:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177032.396475:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177032.396483:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177032.396485:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177032.396489:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177032.396493:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177032.396499:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11706 00000400:00000200:3.0:1786177032.396507:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177032.396512:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177032.396515:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177032.396517:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396520:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177032.396522:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396524:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177032.396527:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177032.396528:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177032.396531:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177032.396533:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177032.396535:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177032.396536:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177032.396538:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396540:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177032.396541:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396543:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177032.396545:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177032.396547:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396549:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177032.396551:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177032.396553:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177032.396555:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.396557:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177032.396559:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177032.396567:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786177032.396570:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177032.396573:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177032.396575:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.396577:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177032.396579:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177032.396582:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177032.396588:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786177032.396594:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177032.396601:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177032.396609:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177032.396612:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177032.396615:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177032.396633:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177032.396637:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177032.396640:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177032.396642:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177032.396644:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177032.396646:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177032.396649:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11706 00000400:00000200:3.0:1786177032.396653:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177032.396656:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177032.396659:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177032.396661:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396664:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177032.396666:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396668:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177032.396671:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177032.396672:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000100:00000001:1.0:1786177032.396673:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1786177032.396674:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177032.396677:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1786177032.396677:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177032.396679:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177032.396680:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1786177032.396681:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177032.396684:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396686:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177032.396687:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177032.396688:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396690:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177032.396693:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:1.0:1786177032.396696:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1786177032.396696:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177032.396699:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.396699:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177032.396701:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177032.396703:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177032.396705:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177032.396706:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177032.396708:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177032.396708:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.396711:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:1.0:1786177032.396712:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235566720:192.168.202.132@tcp:250:kworker.0 00000008:00100000:3.0:1786177032.396715:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1786177032.396716:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:3.0:1786177032.396718:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.396720:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1786177032.396720:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.396723:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177032.396724:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 02000000:00000001:1.0:1786177032.396725:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1786177032.396726:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177032.396728:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000010:1.0:1786177032.396730:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:3.0:1786177032.396730:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.396732:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177032.396733:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177032.396733:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177032.396735:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177032.396738:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:3.0:1786177032.396738:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000400:00000010:1.0:1786177032.396742:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00000001:3.0:1786177032.396742:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177032.396747:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177032.396749:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000001:3.0:1786177032.396754:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1786177032.396756:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235566720, portal 25 00000100:00000001:3.0:1786177032.396756:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.396758:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177032.396760:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280821632 : -110371428729984 : ffff9b9e26327b80) 00000100:00000040:3.0:1786177032.396761:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177032.396766:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.396775:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177032.396778:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396778:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177032.396781:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00080000:3.0:1786177032.396781:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1786177032.396784:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235566720, offset 0 00000100:00080000:3.0:1786177032.396784:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177032.396786:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177032.396787:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177032.396790:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177032.396791:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177032.396793:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11706 00000400:00000200:3.0:1786177032.396796:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177032.396798:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177032.396799:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177032.396801:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177032.396803:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000040:3.0:1786177032.396804:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396806:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177032.396808:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177032.396812:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:3.0:1786177032.396815:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177032.396816:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1786177032.396817:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177032.396819:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000001:3.0:1786177032.396820:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177032.396821:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000400:00000200:1.0:1786177032.396823:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426494:426494:256:4294967295] 192.168.202.132@tcp LPNI seq info [426456:426456:8:4294967295] 00000020:00000001:3.0:1786177032.396823:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177032.396825:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177032.396827:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177032.396829:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:3.0:1786177032.396830:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177032.396832:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396835:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177032.396837:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177032.396839:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:3.0:1786177032.396839:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177032.396842:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177032.396844:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:1.0:1786177032.396846:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396847:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177032.396849:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:1.0:1786177032.396851:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 02000000:00000001:3.0:1786177032.396851:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177032.396854:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.396855:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:1.0:1786177032.396858:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000008:00100000:3.0:1786177032.396858:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177032.396860:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.396862:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177032.396863:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000010:3.0:1786177032.396866:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786177032.396868:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177032.396870:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.1:1786177032.396870:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786177032.396871:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.396873:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1786177032.396874:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 02000000:00000001:3.0:1786177032.396875:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177032.396877:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1786177032.396879:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 02000000:00000010:3.0:1786177032.396880:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000800:00000010:1.0:1786177032.396882:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000100:00000001:3.0:1786177032.396883:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177032.396888:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1786177032.396888:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177032.396892:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1786177032.396896:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177032.396897:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177032.396898:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177032.396900:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000040:3.0:1786177032.396901:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.396904:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177032.396908:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177032.396909:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177032.396912:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177032.396916:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177032.396917:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177032.396918:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1786177032.396919:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1786177032.396922:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177032.396922:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177032.396946:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.396948:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177032.396954:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177032.396955:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11706 00000400:00000200:1.0:1786177032.396958:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177032.396959:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177032.396961:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306383104 00000400:00000200:3.0:1786177032.396962:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177032.396964:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000010:1.0:1786177032.396966:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62e40. 00000100:00000040:3.0:1786177032.396968:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396969:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177032.396972:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177032.396973:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1786177032.396974:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177032.396976:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177032.396978:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177032.396979:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177032.396980:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306383104 00000020:00000040:3.0:1786177032.396981:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000010:1.0:1786177032.396983:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62a80. 00000020:00000001:3.0:1786177032.396984:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177032.396986:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177032.396987:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000040:3.0:1786177032.396989:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177032.396990:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1786177032.396991:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177032.396993:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177032.396993:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.396995:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177032.396996:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000020:00000040:3.0:1786177032.396998:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00100000:1.0:1786177032.397000:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177032/real 1786177032] req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177032.397000:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177032.397003:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177032.397005:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177032.397008:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177032.397011:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.397013:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177032.397015:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000001:1.0:1786177032.397018:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:3.0:1786177032.397018:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1786177032.397020:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1786177032.397020:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177032.397022:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000100:00000010:3.0:1786177032.397023:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000400:00000200:1.0:1786177032.397025:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000020:00000040:3.0:1786177032.397026:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000400:00000010:1.0:1786177032.397027:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 02000000:00000001:3.0:1786177032.397028:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177032.397029:0:2392:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1786177032.397030:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.397032:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1786177032.397033:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177032.397033:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177032.397035:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177032.397039:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000200:1.0:1786177032.397042:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177032.397043:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177032.397047:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177032.397050:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177032.397052:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397055:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1786177032.397055:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177032.397057:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177032.397057:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177032.397061:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177032.397062:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397071:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177032.397071:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177032.397074:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1786177032.397076:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177032.397077:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177032.397079:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177032.397081:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177032.397084:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177032.397084:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177032.397086:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11706 00000100:00000001:1.0:1786177032.397087:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397089:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:3.0:1786177032.397089:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177032.397091:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1786177032.397092:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177032.397093:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1786177032.397094:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177032.397095:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1786177032.397097:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00080000:1.0:1786177032.397098:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1786177032.397099:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177032.397101:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00080000:1.0:1786177032.397102:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:3.0:1786177032.397103:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177032.397104:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:3.0:1786177032.397105:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177032.397107:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177032.397108:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1786177032.397109:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177032.397110:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177032.397112:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177032.397114:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177032.397117:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00100000:1.0:1786177032.397120:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235566720:192.168.202.132@tcp:250:kworker.0 00000100:00000001:3.0:1786177032.397120:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177032.397122:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177032.397124:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177032.397126:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177032.397129:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177032.397132:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177032.397133:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177032.397135:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1786177032.397136:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177032.397137:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1786177032.397138:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.397140:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:1.0:1786177032.397141:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327b80 x1872930235566720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1786177032.397142:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177032.397144:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.397147:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.397149:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000010:3.0:1786177032.397150:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 02000000:00000001:1.0:1786177032.397151:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:3.0:1786177032.397152:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000010:1.0:1786177032.397154:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:3.0:1786177032.397154:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177032.397156:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.397157:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1786177032.397157:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:1.0:1786177032.397159:0:2392:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:3.0:1786177032.397159:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177032.397160:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177032.397161:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177032.397163:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177032.397165:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa000. 02000000:00000010:3.0:1786177032.397165:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786177032.397167:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.397169:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177032.397171:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177032.397172:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327b80. 00000100:00100000:3.0:1786177032.397172:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177032.397176:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177032.397177:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177032.397179:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177032.397180:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177032.397181:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177032.397184:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397187:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177032.397189:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.397192:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1786177032.397192:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.397193:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177032.397195:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1786177032.397196:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11716) 00000100:00000040:1.0:1786177032.397198:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397205:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177032.397206:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.397209:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177032.397210:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177032.397212:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177032.397215:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566848:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177032.397218:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177032.397220:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177032.397222:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.397224:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177032.397226:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177032.397228:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177032.397232:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177032.397235:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177032.397238:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177032.397242:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235566848, portal 10 00000100:00000001:1.0:1786177032.397244:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177032.397245:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280812672 : -110371428738944 : ffff9b9e26325880) 00000100:00000040:1.0:1786177032.397250:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397255:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177032.397256:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177032.397258:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177032.397260:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235566848, offset 0 00000400:00000200:1.0:1786177032.397262:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177032.397265:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177032.397268:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177032.397270:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177032.397272:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177032.397273:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177032.397275:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177032.397277:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426495:426495:256:4294967295] 192.168.202.132@tcp LPNI seq info [426457:426457:8:4294967295] 00000400:00000200:1.0:1786177032.397280:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177032.397285:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177032.397288:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177032.397290:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177032.397293:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177032.397295:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786177032.397298:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177032.397300:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1786177032.397303:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177032.397305:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177032.397307:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177032.397309:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177032.397311:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177032.397313:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177032.397314:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177032.397316:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397322:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177032.397324:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397328:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.397329:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177032.397331:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177032.397334:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177032.397336:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177032.397337:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306379267 00000100:00000001:1.0:1786177032.397339:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306383104 00000100:00000001:1.0:1786177032.397340:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306379267 00000100:00000001:1.0:1786177032.397342:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306383104 00000100:00000001:1.0:1786177032.397343:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177032.397345:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.397346:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177032.397348:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177032/real 1786177032] req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177032.397383:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177032.397384:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177032.397385:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177032.397388:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177032.397390:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177032.397393:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177032.397397:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177032.397406:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397413:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177032.397415:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397417:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177032.397418:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177032.397423:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397431:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177032.397435:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397443:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177032.397445:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397447:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177032.397448:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397449:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177032.397451:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177032.397453:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177032.397455:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177032.397456:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177032.397459:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177032.397465:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566848:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177032.397468:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177032.397470:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397475:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177032.397476:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.397477:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177032.397479:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177032.397480:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177032.397482:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566976:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177032.397484:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177032.397486:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177032.397487:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.397488:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177032.397491:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177032.397492:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177032.397495:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177032.397496:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177032.397498:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177032.397501:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235566976, portal 10 00000100:00000001:1.0:1786177032.397502:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177032.397503:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280811776 : -110371428739840 : ffff9b9e26325500) 00000100:00000040:1.0:1786177032.397507:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397511:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177032.397513:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177032.397514:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177032.397516:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235566976, offset 0 00000400:00000200:1.0:1786177032.397518:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177032.397521:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177032.397523:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177032.397525:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177032.397527:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177032.397529:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177032.397531:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177032.397533:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426496:426496:256:4294967295] 192.168.202.132@tcp LPNI seq info [426458:426458:8:4294967295] 00000400:00000200:1.0:1786177032.397535:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177032.397540:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177032.397542:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177032.397544:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177032.397547:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177032.397549:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786177032.397552:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177032.397554:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1786177032.397556:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177032.397558:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177032.397560:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177032.397563:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177032.397565:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177032.397566:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177032.397568:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177032.397570:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397575:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177032.397577:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397581:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.397582:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177032.397584:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177032.397586:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177032.397588:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177032.397590:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306379268 00000100:00000001:1.0:1786177032.397591:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306383104 00000100:00000001:1.0:1786177032.397593:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306379268 00000100:00000001:1.0:1786177032.397595:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306383104 00000100:00000001:1.0:1786177032.397596:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177032.397598:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.397599:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177032.397601:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177032/real 1786177032] req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177032.397606:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177032.397607:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177032.397607:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177032.397609:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177032.397610:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177032.397611:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177032.397613:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177032.397618:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397623:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177032.397623:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397624:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177032.397625:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177032.397627:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397631:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177032.397633:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397637:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177032.397639:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397640:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177032.397641:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397642:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177032.397644:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177032.397645:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177032.397648:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177032.397649:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177032.397651:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177032.397656:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235566976:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177032.397658:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177032.397660:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397664:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177032.397665:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.397666:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177032.397667:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177032.397668:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177032.397671:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567104:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177032.397673:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177032.397674:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177032.397675:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.397676:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177032.397678:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177032.397679:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177032.397681:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177032.397682:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177032.397684:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177032.397686:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235567104, portal 4 00000100:00000001:1.0:1786177032.397687:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177032.397688:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280808192 : -110371428743424 : ffff9b9e26324700) 00000100:00000040:1.0:1786177032.397692:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397697:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177032.397698:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177032.397700:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177032.397701:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235567104, offset 0 00000400:00000200:1.0:1786177032.397703:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177032.397705:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177032.397707:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177032.397709:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177032.397710:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177032.397712:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177032.397713:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177032.397715:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426497:426497:256:4294967295] 192.168.202.132@tcp LPNI seq info [426459:426459:8:4294967295] 00000400:00000200:1.0:1786177032.397718:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177032.397722:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177032.397725:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177032.397727:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177032.397729:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177032.397731:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786177032.397733:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177032.397735:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1786177032.397738:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177032.397739:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177032.397741:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177032.397743:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177032.397745:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177032.397746:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177032.397747:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177032.397749:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397754:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177032.397756:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397764:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.397766:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177032.397767:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177032.397769:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177032.397771:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177032.397772:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306379268 00000100:00000001:1.0:1786177032.397774:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306383104 00000100:00000001:1.0:1786177032.397775:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306379268 00000100:00000001:1.0:1786177032.397776:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306383104 00000100:00000001:1.0:1786177032.397778:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177032.397779:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.397780:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177032.397782:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177032/real 1786177032] req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177032.397786:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177032.397787:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177032.397788:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177032.397789:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177032.397790:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177032.397791:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177032.397793:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177032.397798:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397802:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177032.397803:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397804:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177032.397805:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177032.397807:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397812:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177032.397815:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397819:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177032.397820:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397821:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177032.397822:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397823:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177032.397824:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177032.397825:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177032.397827:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177032.397828:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177032.397831:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177032.397835:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567104:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177032.397837:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177032.397839:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397843:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177032.397844:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.397845:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177032.397846:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177032.397847:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177032.397849:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567232:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177032.397850:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177032.397851:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177032.397853:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177032.397854:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177032.397856:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af000. 02000000:00000001:1.0:1786177032.397857:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177032.397859:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177032.397861:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177032.397863:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177032.397864:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235567232, portal 4 00000100:00000001:1.0:1786177032.397865:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177032.397866:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809088 : -110371428742528 : ffff9b9e26324a80) 00000100:00000040:1.0:1786177032.397870:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397874:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177032.397875:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177032.397877:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177032.397878:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235567232, offset 0 00000400:00000200:1.0:1786177032.397880:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177032.397882:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177032.397884:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177032.397886:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177032.397887:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177032.397888:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177032.397890:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177032.397892:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426498:426498:256:4294967295] 192.168.202.132@tcp LPNI seq info [426460:426460:8:4294967295] 00000400:00000200:1.0:1786177032.397895:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177032.397899:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177032.397902:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177032.397903:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177032.397905:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177032.397907:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786177032.397909:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177032.397911:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1786177032.397913:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177032.397915:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177032.397917:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177032.397919:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177032.397921:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:1.0:1786177032.397922:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:1.0:1786177032.397937:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177032.397940:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397946:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177032.397947:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177057 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397952:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.397953:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177032.397955:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177032.397957:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177032.397959:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177032.397961:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306379270 00000100:00000001:1.0:1786177032.397962:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306383104 00000100:00000001:1.0:1786177032.397964:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306379270 00000100:00000001:1.0:1786177032.397965:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306383104 00000100:00000001:1.0:1786177032.397967:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177032.397968:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.397969:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177032.397971:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177032/real 1786177032] req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177032.397976:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177032.397977:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177032.397978:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177032.397979:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177032.397981:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177032.397982:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177032.397984:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177032.397989:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.397993:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177032.397994:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.397995:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177032.397996:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177032.397998:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.398002:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177032.398004:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.398008:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177032.398010:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.398011:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177032.398011:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.398013:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177032.398014:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177032.398015:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177032.398017:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177032.398019:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177032.398021:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177032.398026:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567232:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177032.398028:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.398029:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177032.398031:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325880 x1872930235566848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.398036:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177032.398037:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177032.398039:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177032.398040:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177032.398041:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177032.398042:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177032.398043:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177032.398045:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa800. 02000000:00000001:1.0:1786177032.398047:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177032.398047:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177032.398049:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325880. 00000100:00000001:1.0:1786177032.398050:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177032.398051:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.398053:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177032.398055:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325500 x1872930235566976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.398058:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177032.398059:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177032.398060:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177032.398061:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177032.398062:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177032.398063:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177032.398064:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177032.398064:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9000. 02000000:00000001:1.0:1786177032.398066:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177032.398067:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177032.398067:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325500. 00000100:00000001:1.0:1786177032.398069:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177032.398070:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.398071:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177032.398073:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324700 x1872930235567104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.398077:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177032.398078:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177032.398079:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177032.398080:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177032.398081:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177032.398082:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177032.398083:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177032.398084:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9400. 02000000:00000001:1.0:1786177032.398085:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177032.398085:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177032.398086:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324700. 00000100:00000001:1.0:1786177032.398088:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177032.398088:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.398089:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177032.398091:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324a80 x1872930235567232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177057 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177032.398095:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177032.398095:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177032.398096:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af000. 02000000:00000001:1.0:1786177032.398097:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177032.398098:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177032.398099:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177032.398100:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177032.398100:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:1.0:1786177032.398102:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177032.398102:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177032.398103:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324a80. 00000100:00000001:1.0:1786177032.398104:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177032.398105:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.398106:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177032.398108:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177032.398109:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177032.398110:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177032.398112:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177032.908459:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177032.908468:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177032.908471:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177032.908477:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177032.908493:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177032.908498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177032.908501:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177032.908504:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177032.908508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177033.292477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177033.292482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177033.292484:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177033.292490:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177033.292490:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177033.292497:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177033.292499:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177033.292501:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177033.292503:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177033.292505:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177033.292508:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177033.292509:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177033.292510:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177033.292513:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177033.292514:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177033.292584:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177033.292589:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177033.292594:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177033.932490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177033.932496:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177033.932498:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177033.932503:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177033.932508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177033.932512:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177033.932514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177033.932516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177033.932518:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177034.316432:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177034.316436:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177034.316439:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177034.316440:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177034.316442:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177034.316446:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177034.316447:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177034.316448:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177034.316451:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177034.316452:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177034.316455:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177034.316455:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177034.316458:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177034.316460:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177034.316462:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177034.316505:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177034.316508:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177034.316512:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177034.956433:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177034.956440:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177034.956442:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177034.956447:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177034.956450:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177034.956452:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177034.956455:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177034.956457:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177034.956460:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177035.340469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177035.340476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177035.340477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177035.340478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177035.340482:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177035.340483:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177035.340488:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177035.340490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177035.340491:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177035.340494:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177035.340495:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177035.340498:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177035.340501:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177035.340502:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177035.340505:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177035.340571:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177035.340575:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177035.340579:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177035.980518:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177035.980528:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177035.980531:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177035.980537:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177035.980542:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177035.980546:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177035.980549:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177035.980552:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177035.980566:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177036.300491:0:206554:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306383104, at 4306383168 00000100:00000040:0.0:1786177036.300495:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306383104, at 4306383168 00000100:00000010:1.0:1786177036.300508:0:206554:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62a80. 00000100:00000010:0.0:1786177036.300510:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62e40. 00000100:00000001:1.0:1786177036.364468:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177036.364474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177036.364475:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177036.364477:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177036.364480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177036.364483:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177036.364486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177036.364487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177036.364491:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177036.364492:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177036.364496:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177036.364498:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177036.364499:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177036.364502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177036.364505:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177036.364597:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177036.364600:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177036.364602:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177037.004429:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177037.004438:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177037.004442:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177037.004470:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177037.004475:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177037.004478:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177037.004482:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177037.004484:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177037.004488:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.388483:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177037.388490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177037.388493:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177037.388495:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177037.388500:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177037.388502:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177037.388507:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177037.388508:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177037.388510:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177037.388512:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.388514:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177037.388518:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177037.388521:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177037.388521:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177037.388526:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177037.388578:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177037.388582:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177037.388585:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177037.516489:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177037.516498:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177037.516501:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177037.516504:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177037.516507:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177037.516514:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11711 00000400:00000200:3.0:1786177037.516521:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177037.516526:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177037.516529:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177037.516531:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516534:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177037.516536:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516538:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177037.516541:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177037.516543:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177037.516545:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177037.516548:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177037.516549:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177037.516551:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177037.516553:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516554:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177037.516556:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516557:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177037.516560:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177037.516562:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516564:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177037.516566:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177037.516569:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177037.516571:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.516573:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177037.516575:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177037.516583:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786177037.516585:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177037.516588:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177037.516589:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.516592:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177037.516594:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177037.516598:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177037.516604:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177037.516609:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177037.516617:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177037.516624:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177037.516628:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177037.516632:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177037.516650:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177037.516652:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177037.516655:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177037.516656:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177037.516657:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177037.516659:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177037.516660:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11711 00000400:00000200:3.0:1786177037.516663:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177037.516664:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177037.516666:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177037.516668:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516669:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177037.516671:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516673:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177037.516675:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177037.516676:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177037.516677:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177037.516678:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177037.516680:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177037.516682:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177037.516684:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516685:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177037.516687:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516688:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177037.516690:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177037.516692:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516693:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.516695:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1786177037.516695:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177037.516700:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177037.516701:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1786177037.516703:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.516707:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177037.516707:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177037.516712:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177037.516716:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.516719:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177037.516720:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1786177037.516723:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786177037.516729:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177037.516735:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177037.516738:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.516740:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177037.516743:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177037.516744:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177037.516747:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177037.516750:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1786177037.516751:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.516755:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1786177037.516758:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 02000000:00000001:1.0:1786177037.516761:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177037.516766:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235567360:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177037.516771:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177037.516774:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1786177037.516777:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.516779:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.516783:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1786177037.516786:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1786177037.516791:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 00000100:00000001:3.0:1786177037.516795:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1786177037.516796:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.516799:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177037.516803:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177037.516807:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177037.516815:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00000001:3.0:1786177037.516816:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177037.516821:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177037.516823:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:1.0:1786177037.516825:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000001:3.0:1786177037.516825:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177037.516827:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177037.516831:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177037.516833:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11711 00000100:00000200:1.0:1786177037.516836:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235567360, portal 25 00000400:00000200:3.0:1786177037.516838:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177037.516841:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000400:00000200:3.0:1786177037.516842:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177037.516846:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280806400 : -110371428745216 : ffff9b9e26324000) 00000100:00080000:3.0:1786177037.516847:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177037.516851:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516854:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177037.516858:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177037.516861:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177037.516861:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177037.516866:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177037.516868:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177037.516871:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177037.516887:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177037.516889:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177037.516892:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177037.516899:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177037.516901:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1786177037.516902:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177037.516908:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516912:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:1.0:1786177037.516913:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177037.516916:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000010:1.0:1786177037.516920:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00080000:3.0:1786177037.516921:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177037.516926:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177037.516928:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235567360, offset 0 02000000:00000001:3.0:1786177037.516928:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177037.516932:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177037.516935:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.516936:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177037.516938:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177037.516942:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000008:00000001:3.0:1786177037.516943:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.516947:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177037.516951:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786177037.516952:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177037.516957:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:3.0:1786177037.516958:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177037.516961:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.516965:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177037.516966:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1786177037.516969:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 02000000:00000001:3.0:1786177037.516970:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177037.516978:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000400:00000200:1.0:1786177037.516979:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1786177037.516984:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177037.516986:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177037.516994:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00100000:3.0:1786177037.516994:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177037.517001:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177037.517011:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426499:426499:256:4294967295] 192.168.202.132@tcp LPNI seq info [426461:426461:8:4294967295] 00000100:00000001:3.0:1786177037.517013:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177037.517017:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177037.517021:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177037.517022:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177037.517030:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177037.517035:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177037.517041:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00080000:3.0:1786177037.517041:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177037.517045:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177037.517047:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1786177037.517053:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177037.517059:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177037.517066:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177037.517072:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177037.517090:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177037.517096:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177037.517097:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177037.517100:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306388224 00000100:00080000:3.0:1786177037.517104:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11711 00000100:00000010:1.0:1786177037.517109:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62a80. 00000400:00000200:3.0:1786177037.517113:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177037.517119:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000400:00000200:3.0:1786177037.517120:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177037.517127:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177037.517132:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306388224 00000100:00000040:3.0:1786177037.517134:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000010:1.0:1786177037.517139:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62660. 00000100:00000001:3.0:1786177037.517139:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177037.517145:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177037.517147:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1786177037.517149:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177037.517153:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1786177037.517154:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177037.517158:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:1.0:1786177037.517160:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177037.517162:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177037.517166:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1786177037.517167:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:1.0:1786177037.517172:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1786177037.517173:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177037.517179:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177037.517182:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1786177037.517183:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177037.517187:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1786177037.517188:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177037.517192:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1786177037.517194:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177037.517195:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177037.517200:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177037.517200:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177037.517206:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000040:1.0:1786177037.517209:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177037.517214:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177037.517223:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.517227:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1786177037.517228:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177037.517231:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177037.517234:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177037.517235:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1786177037.517240:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.517241:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1786177037.517246:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1786177037.517247:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1786177037.517252:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1786177037.517254:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567488:192.168.202.132@tcp:38:kworker.0 00000008:00000001:3.0:1786177037.517257:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.517263:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1786177037.517263:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.517268:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:3.0:1786177037.517270:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 02000000:00000001:1.0:1786177037.517272:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.517275:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177037.517279:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177037.517281:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177037.517288:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:3.0:1786177037.517290:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177037.517294:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1786177037.517295:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 02000000:00000001:3.0:1786177037.517295:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.517299:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177037.517301:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1786177037.517303:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 02000000:00000001:3.0:1786177037.517304:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177037.517309:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235567488, portal 10 02000000:00000010:3.0:1786177037.517312:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:1.0:1786177037.517313:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177037.517316:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280817152 : -110371428734464 : ffff9b9e26326a00) 00000100:00000001:3.0:1786177037.517319:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177037.517328:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1786177037.517337:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177037.517340:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177037.517345:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.517352:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1786177037.517354:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177037.517384:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177037.517388:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177037.517393:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235567488, offset 0 00000400:00000200:1.0:1786177037.517406:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177037.517408:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177037.517412:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177037.517414:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:3.0:1786177037.517416:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177037.517419:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177037.517422:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1786177037.517423:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:3.0:1786177037.517425:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177037.517429:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1786177037.517432:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11711 00000400:00000200:1.0:1786177037.517436:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:3.0:1786177037.517441:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177037.517443:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1786177037.517450:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177037.517451:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00080000:3.0:1786177037.517456:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177037.517460:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426500:426500:255:4294967295] 192.168.202.132@tcp LPNI seq info [426462:426462:7:4294967295] 00000100:00000040:3.0:1786177037.517464:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.517469:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177037.517470:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177037.517476:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.517479:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177037.517485:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1786177037.517488:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000020:00000040:3.0:1786177037.517489:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177037.517494:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177037.517498:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000800:00000200:1.0:1786177037.517502:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000020:00000001:3.0:1786177037.517503:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177037.517508:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000800:00000010:1.0:1786177037.517509:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e200. 00000100:00000040:3.0:1786177037.517514:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:1.0:1786177037.517521:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177037.517521:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177037.517527:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177037.517533:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:1.1:1786177037.517534:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786177037.517539:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:1.0:1786177037.517541:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177037.517547:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000040:1.0:1786177037.517548:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000100:00000001:3.0:1786177037.517554:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177037.517556:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:3.0:1786177037.517558:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177037.517561:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306384388 02000000:00000001:3.0:1786177037.517564:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177037.517567:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306388224 02000000:00000001:3.0:1786177037.517569:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.517572:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306384388 00000100:00000001:3.0:1786177037.517574:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177037.517578:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306388224 00000008:00100000:3.0:1786177037.517580:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177037.517583:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1786177037.517585:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.517588:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.517590:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.517593:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:3.0:1786177037.517597:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000100:00000040:1.0:1786177037.517601:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177037.517603:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177037.517610:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177037.517612:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.517618:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177037.517619:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1786177037.517620:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177037.517622:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177037.517623:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.517628:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:3.0:1786177037.517630:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 02000000:00000001:1.0:1786177037.517632:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177037.517636:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.517637:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177037.517642:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567616:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177037.517649:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1786177037.517649:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177037.517653:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177037.517656:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.517661:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177037.517667:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 00000100:00000001:3.0:1786177037.517668:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1786177037.517673:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177037.517674:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177037.517680:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177037.517683:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1786177037.517686:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177037.517693:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000001:3.0:1786177037.517696:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177037.517698:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235567616, portal 10 00000100:00000001:1.0:1786177037.517702:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1786177037.517703:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11722) 00000100:00000001:1.0:1786177037.517704:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809984 : -110371428741632 : ffff9b9e26324e00) 00000100:00000040:1.0:1786177037.517715:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177037.517723:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177037.517727:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177037.517730:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e220. 00000100:00000200:1.0:1786177037.517732:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235567616, offset 0 00000400:00000200:1.0:1786177037.517737:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177037.517741:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177037.517745:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177037.517748:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177037.517751:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177037.517754:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177037.517756:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177037.517760:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426501:426501:254:4294967295] 192.168.202.132@tcp LPNI seq info [426463:426463:6:4294967295] 00000400:00000200:1.0:1786177037.517765:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177037.517771:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177037.517776:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177037.517780:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142ec00. 00000800:00000200:1.0:1786177037.517784:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177037.517789:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177037.517792:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177037.517795:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177037.517798:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177037.517800:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306384388 00000100:00000001:1.0:1786177037.517803:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306388224 00000100:00000001:1.0:1786177037.517806:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306384388 00000100:00000001:1.0:1786177037.517809:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306388224 00000100:00000001:1.0:1786177037.517811:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177037.517813:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.517816:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.517818:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177037.517822:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177037.517824:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1786177037.517826:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177037.517827:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177037.517829:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177037.517833:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177037.517841:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177037.517843:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.517845:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177037.517847:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177037.517849:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177037.517852:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567744:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177037.517856:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177037.517858:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177037.517859:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.517861:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177037.517863:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177037.517865:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177037.517869:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177037.517871:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177037.517888:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ed48. 00000100:00000200:1.0:1786177037.517892:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235567744, portal 4 00000100:00000001:1.0:1786177037.517893:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177037.517894:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809088 : -110371428742528 : ffff9b9e26324a80) 00000100:00000040:1.0:1786177037.517899:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177037.517904:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177037.517906:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177037.517908:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177037.517910:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235567744, offset 0 00000400:00000200:1.0:1786177037.517912:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177037.517915:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177037.517918:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177037.517920:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177037.517922:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177037.517924:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177037.517925:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177037.517928:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426502:426502:253:4294967295] 192.168.202.132@tcp LPNI seq info [426464:426464:5:4294967295] 00000400:00000200:1.0:1786177037.517930:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177037.517934:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177037.517937:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177037.517939:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e000. 00000800:00000200:1.0:1786177037.517942:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177037.517945:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177037.517948:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177037.517950:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177037.517954:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177037.517956:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306384388 00000100:00000001:1.0:1786177037.517958:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306388224 00000100:00000001:1.0:1786177037.517961:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306384388 00000100:00000001:1.0:1786177037.517963:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306388224 00000100:00000001:1.0:1786177037.517965:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177037.517967:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.517970:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177037.517973:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177037.517981:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177037.517982:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.517985:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177037.517987:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177037.517989:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177037.517992:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567872:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177037.517995:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177037.517997:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177037.517999:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177037.518000:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177037.518004:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad800. 02000000:00000001:1.0:1786177037.518005:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177037.518008:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177037.518011:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842d68. 00000400:00000010:1.0:1786177037.518015:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027edd0. 00000100:00000200:1.0:1786177037.518018:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235567872, portal 4 00000100:00000001:1.0:1786177037.518020:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177037.518021:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280808192 : -110371428743424 : ffff9b9e26324700) 00000100:00000040:1.0:1786177037.518027:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177037.518035:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177037.518037:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177037.518040:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e5d8. 00000100:00000200:1.0:1786177037.518042:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235567872, offset 0 00000400:00000200:1.0:1786177037.518047:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177037.518051:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177037.518055:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177037.518059:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177037.518064:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177037.518066:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177037.518068:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177037.518071:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426503:426503:252:4294967295] 192.168.202.132@tcp LPNI seq info [426465:426465:4:4294967295] 00000400:00000200:1.0:1786177037.518075:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177037.518081:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177037.518087:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177037.518091:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177037.518094:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177037.518098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177037.518100:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177037.518104:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177037.518107:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177037.518109:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306384390 00000100:00000001:1.0:1786177037.518111:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306388224 00000100:00000001:1.0:1786177037.518114:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306384390 00000100:00000001:1.0:1786177037.518116:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306388224 00000100:00000001:1.0:1786177037.518118:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177037.518121:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.518123:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.518125:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177037.518129:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177037.518131:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1786177037.518132:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177037.518134:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177037.518136:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177037.518137:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177038.028450:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177038.028457:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177038.028460:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177038.028464:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177038.028468:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177038.028471:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177038.028474:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177038.028476:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177038.028479:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177038.412454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177038.412461:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177038.412463:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177038.412468:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177038.412470:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177038.412471:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177038.412474:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177038.412476:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177038.412480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177038.412482:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177038.412483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177038.412488:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177038.412488:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177038.412491:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177038.412496:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177038.412558:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177038.412562:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177038.412566:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177039.052480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177039.052493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177039.052495:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177039.052500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177039.052504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177039.052507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177039.052509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177039.052511:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177039.052514:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177039.436473:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177039.436473:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177039.436482:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177039.436485:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177039.436492:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177039.436496:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177039.436500:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177039.436503:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177039.436506:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177039.436511:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177039.436513:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177039.436516:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177039.436521:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177039.436523:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177039.436525:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177039.436528:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177039.436530:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177039.436531:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177039.436533:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177039.436535:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177039.436536:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177039.436582:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177039.436586:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177039.436589:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177040.076514:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177040.076527:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177040.076530:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177040.076537:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177040.076542:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177040.076548:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177040.076551:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177040.076555:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177040.076559:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177040.460446:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177040.460449:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177040.460455:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177040.460457:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177040.460462:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177040.460465:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177040.460469:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177040.460469:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177040.460474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177040.460476:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177040.460476:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177040.460483:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.460488:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177040.460491:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.460494:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177040.460539:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177040.460544:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177040.460548:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1786177040.588699:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:0.0:1786177040.588743:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1786177040.588753:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177040.588757:0:2380:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:0.0:1786177040.588762:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177040.588766:0:2380:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1786177040.588771:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177040.588775:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:0.0:1786177040.588781:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177040.588786:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177040.588793:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000400:00000010:0.0:1786177040.588796:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:0.0:1786177040.588801:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177040.588810:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177040.588887:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177040.588893:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177040.588909:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177040.588911:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786177040.588922:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177040.588925:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:0.0:1786177040.588929:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177040.588933:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177040.588937:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:0.0:1786177040.588938:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:0.0:1786177040.588941:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177040.588946:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177040.588955:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177040.588958:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177040.588963:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177040.588965:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786177040.588972:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177040.588976:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:0.0:1786177040.588980:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177040.588984:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177040.588988:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e220 00000400:00000010:0.0:1786177040.588990:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e220. 00000100:00000001:0.0:1786177040.588993:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177040.588998:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177040.589007:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177040.589011:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177040.589019:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177040.589021:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786177040.589028:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177040.589031:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:0.0:1786177040.589034:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177040.589038:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177040.589041:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:0.0:1786177040.589044:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:0.0:1786177040.589046:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177040.589052:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177040.589060:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177040.589065:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177040.589072:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177040.589074:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786177040.589080:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177040.589082:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:0.0:1786177040.589085:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177040.589089:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177040.589092:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e5d8 00000400:00000010:0.0:1786177040.589095:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e5d8. 00000100:00000001:0.0:1786177040.589098:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177040.589103:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177040.589111:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177040.589115:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177062 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177040.589123:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177040.589125:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177040.589226:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177040.589233:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177040.589237:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177040.589246:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177037/real 1786177040] req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177040.589260:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177040.589262:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177040.589266:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177040.589270:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ed48 00000400:00000010:1.0:1786177040.589272:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ed48. 00000100:00000001:1.0:1786177040.589283:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177040.589287:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177040.589298:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589306:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177040.589308:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589311:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177040.589314:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177040.589318:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589333:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177040.589339:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589347:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177040.589351:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589353:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177040.589354:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589384:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177040.589386:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177040.589391:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177040.589397:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177040.589400:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177040.589405:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177040.589414:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567744:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177040.589421:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177040.589425:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177037/real 1786177040] req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177040.589434:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177040.589436:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177040.589438:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842d68. 00000400:00000200:1.0:1786177040.589442:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027edd0 00000400:00000010:1.0:1786177040.589444:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027edd0. 00000100:00000001:1.0:1786177040.589446:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177040.589450:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177040.589466:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589474:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177040.589475:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589477:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177040.589479:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177040.589484:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589491:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177040.589496:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589503:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177040.589506:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589507:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177040.589509:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589511:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177040.589512:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177040.589515:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177040.589520:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177040.589522:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177040.589526:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177040.589535:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567872:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177040.589539:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177040.589543:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177037/real 1786177040] req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177040.589558:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177040.589560:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177040.589563:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177040.589566:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177040.589568:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177040.589570:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177040.589574:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177040.589584:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589591:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177040.589593:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589595:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177040.589596:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177040.589601:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589609:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177040.589613:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589622:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177040.589624:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589626:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177040.589628:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589630:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177040.589633:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177040.589635:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177040.589639:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177040.589642:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177040.589647:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177040.589657:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567488:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177040.589663:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177040.589667:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177037/real 1786177040] req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177040.589675:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177040.589677:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177040.589679:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177040.589682:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177040.589684:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177040.589687:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177040.589691:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177040.589701:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589709:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177040.589713:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589716:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177040.589718:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177040.589723:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589731:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177040.589735:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589744:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177040.589746:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589749:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177040.589751:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589752:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177040.589755:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177040.589759:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177040.589763:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177040.589766:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177040.589771:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177040.589780:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235567616:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177040.589786:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177040.589791:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177037/real 1786177040] req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177040.589800:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177040.589801:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177040.589804:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177040.589806:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177040.589809:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177040.589811:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177040.589814:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177040.589824:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589832:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177040.589833:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589835:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177040.589837:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177040.589841:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589873:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177040.589878:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589886:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177040.589889:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589891:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177040.589893:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589895:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177040.589897:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177040.589900:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177040.589903:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177040.589906:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177040.589910:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177040.589919:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235567360:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177040.589924:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589928:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177040.589932:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324a80 x1872930235567744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589940:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177040.589944:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177040.589950:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177040.589954:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177040.589958:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177040.589960:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177040.589963:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177040.589966:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 02000000:00000001:1.0:1786177040.589970:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177040.589973:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177040.589976:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324a80. 00000100:00000001:1.0:1786177040.589980:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177040.589981:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.589983:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177040.589987:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324700 x1872930235567872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.589994:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177040.589996:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177040.589997:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:1.0:1786177040.590000:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177040.590001:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177040.590003:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177040.590012:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177040.590014:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:1.0:1786177040.590016:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177040.590018:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177040.590020:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324700. 00000100:00000001:1.0:1786177040.590023:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177040.590025:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.590027:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177040.590031:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326a00 x1872930235567488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.590039:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177040.590040:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177040.590042:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177040.590044:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177040.590046:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177040.590047:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177040.590049:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177040.590051:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:1.0:1786177040.590053:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177040.590054:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177040.590056:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326a00. 00000100:00000001:1.0:1786177040.590058:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177040.590060:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.590063:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177040.590066:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324e00 x1872930235567616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.590075:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177040.590076:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177040.590078:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177040.590081:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177040.590082:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177040.590084:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177040.590086:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177040.590087:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:1.0:1786177040.590090:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177040.590091:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177040.590093:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324e00. 00000100:00000001:1.0:1786177040.590095:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177040.590096:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.590098:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177040.590102:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324000 x1872930235567360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177062 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177040.590110:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177040.590111:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177040.590113:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177040.590115:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177040.590116:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177040.590118:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177040.590120:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177040.590122:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:1.0:1786177040.590124:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177040.590125:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177040.590127:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324000. 00000100:00000001:1.0:1786177040.590129:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177040.590130:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.590133:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177040.590144:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177040.590147:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177040.590150:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177040.590152:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177041.100465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177041.100472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177041.100474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177041.100480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177041.100485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177041.100489:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177041.100493:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177041.100496:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177041.100500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177041.420471:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306388224, at 4306388288 00000100:00000040:1.0:1786177041.420471:0:206554:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306388224, at 4306388288 00000100:00000010:0.0:1786177041.420487:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62a80. 00000100:00000010:1.0:1786177041.420489:0:206554:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62660. 00000100:00000001:1.0:1786177041.484466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177041.484470:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177041.484474:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177041.484477:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177041.484478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177041.484482:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1786177041.484485:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177041.484487:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177041.484491:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177041.484493:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177041.484497:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177041.484497:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177041.484503:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177041.484504:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177041.484507:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177041.484509:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177041.484512:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177041.484516:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177041.484520:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177041.484525:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177041.484527:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177041.484529:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177041.484530:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177041.484532:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177041.484533:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177041.484535:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177041.484536:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177041.484583:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177041.484589:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177041.484592:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177042.124508:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177042.124517:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177042.124520:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177042.124525:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177042.124529:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177042.124533:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177042.124536:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177042.124539:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177042.124543:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177042.508441:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177042.508441:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177042.508447:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.508448:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177042.508452:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177042.508453:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177042.508456:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177042.508456:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177042.508462:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177042.508463:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177042.508467:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.508467:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.508473:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177042.508476:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.508478:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177042.508525:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177042.508528:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177042.508531:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786177042.636531:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177042.636540:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786177042.636543:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177042.636545:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177042.636549:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177042.636555:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11717 00000400:00000200:0.0:1786177042.636562:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177042.636567:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177042.636570:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177042.636574:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636577:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177042.636580:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636583:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177042.636587:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177042.636589:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786177042.636592:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177042.636596:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786177042.636599:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786177042.636602:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786177042.636605:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636607:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177042.636609:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636612:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177042.636615:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786177042.636619:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636622:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177042.636624:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177042.636628:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177042.636630:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.636632:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177042.636635:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177042.636642:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786177042.636646:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786177042.636650:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177042.636652:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.636656:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177042.636658:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177042.636663:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177042.636671:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786177042.636677:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177042.636687:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177042.636699:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177042.636704:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177042.636710:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177042.636742:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177042.636745:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177042.636748:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786177042.636750:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177042.636752:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177042.636754:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177042.636757:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11717 00000400:00000200:0.0:1786177042.636762:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177042.636765:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177042.636767:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177042.636770:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636772:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177042.636774:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636777:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177042.636779:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177042.636781:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786177042.636782:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177042.636784:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786177042.636787:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786177042.636789:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786177042.636792:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636794:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177042.636796:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636798:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177042.636800:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786177042.636803:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636806:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177042.636808:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177042.636814:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177042.636816:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.636817:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177042.636821:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786177042.636834:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.636837:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177042.636841:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786177042.636844:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786177042.636846:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177042.636848:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.636850:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177042.636851:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177042.636852:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177042.636857:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786177042.636861:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177042.636865:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177042.636874:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177042.636875:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177042.636879:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177042.636886:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177042.636889:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177042.636892:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786177042.636893:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177042.636895:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177042.636897:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177042.636900:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11717 00000400:00000200:0.0:1786177042.636904:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177042.636907:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177042.636909:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177042.636912:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636914:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177042.636916:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636918:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177042.636920:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177042.636921:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786177042.636922:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177042.636924:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786177042.636926:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786177042.636927:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786177042.636930:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636932:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177042.636933:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636935:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177042.636937:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786177042.636939:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177042.636941:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177042.636943:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177042.636946:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177042.636947:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.636950:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177042.636952:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786177042.636954:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.636955:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177042.636958:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786177042.636960:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786177042.636963:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177042.636964:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.636965:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177042.636966:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177042.636968:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177042.636972:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786177042.636975:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177042.636979:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177042.636986:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177042.636988:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177042.636992:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177042.637014:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177042.637017:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177042.637020:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786177042.637022:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177042.637023:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177042.637026:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177042.637028:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11717 00000400:00000200:0.0:1786177042.637032:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177042.637035:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177042.637037:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177042.637040:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.637042:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177042.637044:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.637046:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177042.637047:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177042.637049:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786177042.637051:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177042.637053:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786177042.637055:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786177042.637057:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786177042.637060:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.637062:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177042.637064:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.637066:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177042.637068:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786177042.637071:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177042.637073:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177042.637074:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177042.637077:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177042.637079:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.637081:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177042.637083:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786177042.637085:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.637087:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177042.637090:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786177042.637092:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177042.637094:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177042.637095:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.637097:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177042.637098:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177042.637100:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177042.637103:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786177042.637106:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177042.637110:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177042.637116:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177042.637118:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177042.637121:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177042.637128:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177042.637130:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177042.637134:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786177042.637135:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177042.637137:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177042.637140:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177042.637142:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11717 00000400:00000200:0.0:1786177042.637146:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177042.637149:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177042.637151:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177042.637154:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.637156:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177042.637158:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.637159:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177042.637161:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177042.637162:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786177042.637164:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177042.637171:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786177042.637174:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786177042.637175:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786177042.637178:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.637180:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177042.637182:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177042.637184:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177042.637186:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786177042.637189:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177042.637191:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177042.637193:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177042.637196:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177042.637198:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.637200:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177042.637202:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786177042.637204:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.637205:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177042.637209:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786177042.637211:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177042.637213:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177042.637215:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177042.637217:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177042.637218:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177042.637220:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177042.637230:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786177042.637233:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177042.637238:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177042.637245:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177042.637252:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177042.637256:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177042.637263:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177042.637267:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11727) 00000100:00000001:1.0:1786177042.637561:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177042.637566:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177042.637569:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177042.637576:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.637586:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177042.637588:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177042.637592:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177042.637595:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177042.637597:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177042.637601:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235568000:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177042.637605:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177042.637610:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177042.637613:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177042.637616:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177042.637622:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177042.637624:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177042.637631:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177042.637635:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177042.637641:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177042.637649:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235568000, portal 25 00000100:00000001:1.0:1786177042.637652:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177042.637653:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794202496 : -110371915349120 : ffff9b9e09314380) 00000100:00000040:1.0:1786177042.637660:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.637668:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177042.637671:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177042.637674:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177042.637677:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235568000, offset 0 00000400:00000200:1.0:1786177042.637682:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177042.637689:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177042.637694:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177042.637697:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177042.637702:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177042.637706:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177042.637709:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177042.637713:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426504:426504:256:4294967295] 192.168.202.132@tcp LPNI seq info [426466:426466:8:4294967295] 00000400:00000200:1.0:1786177042.637719:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177042.637729:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177042.637736:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177042.637741:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177042.637749:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177042.637753:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177042.637759:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177042.637763:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177042.637768:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177042.637771:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177042.637776:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177042.637781:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177042.637787:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177042.637789:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177042.637793:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177042.637797:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.637807:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177042.637811:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.637819:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.637821:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177042.637852:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177042.637856:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177042.637859:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306393345 00000100:00000010:1.0:1786177042.637865:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62660. 00000100:00000001:1.0:1786177042.637871:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177042.637878:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306393345 00000100:00000010:1.0:1786177042.637881:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62b40. 00000100:00000001:1.0:1786177042.637883:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177042.637886:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177042.637889:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.637891:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177042.637896:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177042/real 1786177042] req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177042.637907:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177042.637909:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177042.637911:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177042.637914:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177042.637915:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177042.637918:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177042.637921:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177042.637930:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.637937:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177042.637939:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.637941:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177042.637943:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177042.637947:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.637954:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177042.637959:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.637967:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177042.637970:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.637971:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177042.637972:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.637974:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177042.637976:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177042.637978:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177042.637980:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177042.637982:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177042.637985:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177042.637991:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235568000:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177042.637994:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177042.637996:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638000:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177042.638002:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177042.638003:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177042.638005:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177042.638006:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177042.638008:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568128:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177042.638010:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177042.638012:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177042.638013:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177042.638014:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177042.638016:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177042.638018:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177042.638021:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177042.638023:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177042.638025:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177042.638027:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235568128, portal 10 00000100:00000001:1.0:1786177042.638029:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177042.638030:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794210560 : -110371915341056 : ffff9b9e09316300) 00000100:00000040:1.0:1786177042.638035:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638039:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177042.638041:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177042.638042:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177042.638044:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235568128, offset 0 00000400:00000200:1.0:1786177042.638047:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177042.638051:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177042.638054:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177042.638057:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177042.638060:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177042.638062:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177042.638065:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177042.638068:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426505:426505:256:4294967295] 192.168.202.132@tcp LPNI seq info [426467:426467:8:4294967295] 00000400:00000200:1.0:1786177042.638072:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177042.638079:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177042.638084:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177042.638087:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177042.638091:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177042.638094:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177042.638098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177042.638100:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177042.638104:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177042.638106:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177042.638110:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177042.638113:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177042.638116:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177042.638118:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177042.638120:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177042.638124:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638132:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177042.638135:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638142:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.638144:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177042.638147:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177042.638152:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177042.638155:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177042.638157:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306389508 00000100:00000001:1.0:1786177042.638159:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306393345 00000100:00000001:1.0:1786177042.638161:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306389508 00000100:00000001:1.0:1786177042.638171:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306393345 00000100:00000001:1.0:1786177042.638174:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177042.638176:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.638178:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177042.638182:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177042/real 1786177042] req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177042.638190:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177042.638191:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177042.638193:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177042.638195:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177042.638197:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177042.638199:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177042.638202:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177042.638209:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638216:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177042.638217:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.638219:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177042.638220:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177042.638223:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638229:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177042.638233:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638239:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177042.638241:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.638243:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177042.638244:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.638245:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177042.638248:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177042.638250:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177042.638253:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177042.638256:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177042.638261:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177042.638270:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568128:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177042.638274:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177042.638277:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638283:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177042.638284:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177042.638287:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177042.638289:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177042.638291:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177042.638294:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568256:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177042.638297:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177042.638300:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177042.638302:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177042.638308:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177042.638312:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177042.638314:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177042.638318:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177042.638320:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177042.638323:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177042.638326:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235568256, portal 10 00000100:00000001:1.0:1786177042.638328:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177042.638329:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215936 : -110371915335680 : ffff9b9e09317800) 00000100:00000040:1.0:1786177042.638335:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638342:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177042.638344:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177042.638346:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177042.638348:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235568256, offset 0 00000400:00000200:1.0:1786177042.638352:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177042.638391:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177042.638395:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177042.638398:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177042.638400:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177042.638403:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177042.638405:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177042.638408:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426506:426506:256:4294967295] 192.168.202.132@tcp LPNI seq info [426468:426468:8:4294967295] 00000400:00000200:1.0:1786177042.638412:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177042.638418:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177042.638423:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177042.638427:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177042.638431:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177042.638434:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177042.638438:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177042.638441:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177042.638445:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177042.638446:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177042.638450:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177042.638453:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177042.638456:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177042.638458:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177042.638461:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177042.638465:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638474:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177042.638477:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638484:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.638485:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177042.638488:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177042.638490:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177042.638492:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177042.638494:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306389509 00000100:00000001:1.0:1786177042.638496:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306393345 00000100:00000001:1.0:1786177042.638498:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306389509 00000100:00000001:1.0:1786177042.638500:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306393345 00000100:00000001:1.0:1786177042.638502:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177042.638505:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.638507:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177042.638511:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177042/real 1786177042] req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177042.638519:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177042.638521:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177042.638523:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177042.638526:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177042.638527:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177042.638529:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177042.638532:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177042.638540:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638546:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177042.638547:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.638549:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177042.638550:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177042.638554:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638560:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177042.638564:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638571:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177042.638573:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.638575:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177042.638576:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.638578:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177042.638581:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177042.638583:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177042.638587:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177042.638589:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177042.638593:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177042.638601:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568256:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177042.638605:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177042.638609:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638617:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177042.638619:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177042.638621:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177042.638623:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177042.638626:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177042.638629:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568384:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177042.638632:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177042.638635:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177042.638637:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177042.638639:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177042.638642:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad800. 02000000:00000001:1.0:1786177042.638643:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177042.638647:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177042.638650:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177042.638654:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177042.638657:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235568384, portal 4 00000100:00000001:1.0:1786177042.638659:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177042.638660:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794212352 : -110371915339264 : ffff9b9e09316a00) 00000100:00000040:1.0:1786177042.638666:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638673:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177042.638675:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177042.638679:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177042.638681:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235568384, offset 0 00000400:00000200:1.0:1786177042.638685:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177042.638690:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177042.638693:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177042.638695:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177042.638696:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177042.638698:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177042.638700:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177042.638704:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426507:426507:256:4294967295] 192.168.202.132@tcp LPNI seq info [426469:426469:8:4294967295] 00000400:00000200:1.0:1786177042.638709:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177042.638715:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177042.638720:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177042.638724:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177042.638729:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177042.638732:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177042.638737:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177042.638740:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177042.638745:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177042.638747:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177042.638751:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177042.638755:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177042.638758:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177042.638759:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177042.638762:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177042.638767:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638777:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177042.638780:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638787:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.638789:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177042.638792:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177042.638795:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177042.638799:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177042.638801:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306389509 00000100:00000001:1.0:1786177042.638804:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306393345 00000100:00000001:1.0:1786177042.638806:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306389509 00000100:00000001:1.0:1786177042.638808:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306393345 00000100:00000001:1.0:1786177042.638810:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177042.638812:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.638814:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177042.638819:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177042/real 1786177042] req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177042.638845:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177042.638846:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177042.638848:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177042.638851:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177042.638853:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177042.638855:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177042.638859:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177042.638866:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638870:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177042.638872:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.638873:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177042.638874:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177042.638877:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638882:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177042.638884:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638889:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177042.638890:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.638891:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177042.638892:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.638893:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177042.638894:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177042.638895:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177042.638897:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177042.638899:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177042.638901:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177042.638908:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568384:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177042.638912:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177042.638915:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638921:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177042.638923:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177042.638925:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177042.638927:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177042.638929:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177042.638933:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568512:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177042.638936:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177042.638938:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177042.638940:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177042.638942:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177042.638946:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177042.638948:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177042.638952:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177042.638954:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177042.638958:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177042.638960:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235568512, portal 4 00000100:00000001:1.0:1786177042.638962:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177042.638963:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794208768 : -110371915342848 : ffff9b9e09315c00) 00000100:00000040:1.0:1786177042.638967:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.638972:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177042.638973:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177042.638975:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177042.638976:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235568512, offset 0 00000400:00000200:1.0:1786177042.638979:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177042.638981:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177042.638984:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177042.638986:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177042.638987:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177042.638989:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177042.638991:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177042.638993:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426508:426508:256:4294967295] 192.168.202.132@tcp LPNI seq info [426470:426470:8:4294967295] 00000400:00000200:1.0:1786177042.638996:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177042.639000:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177042.639003:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177042.639005:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177042.639008:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177042.639010:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177042.639013:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177042.639015:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177042.639017:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177042.639019:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177042.639021:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177042.639023:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177042.639025:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177042.639026:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177042.639028:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177042.639030:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.639035:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177042.639037:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177067 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.639042:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.639043:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177042.639045:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177042.639047:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177042.639049:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177042.639050:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306389511 00000100:00000001:1.0:1786177042.639052:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306393345 00000100:00000001:1.0:1786177042.639053:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306389511 00000100:00000001:1.0:1786177042.639055:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306393345 00000100:00000001:1.0:1786177042.639056:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177042.639057:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.639058:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177042.639061:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177042/real 1786177042] req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177042.639066:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177042.639067:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177042.639068:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177042.639069:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177042.639070:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177042.639072:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177042.639074:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177042.639079:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.639082:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177042.639083:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.639084:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177042.639085:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177042.639087:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.639092:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177042.639095:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.639102:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177042.639104:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.639107:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177042.639108:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.639110:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177042.639111:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177042.639114:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177042.639117:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177042.639119:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177042.639123:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177042.639136:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568512:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177042.639141:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.639144:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177042.639147:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314380 x1872930235568000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.639154:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177042.639156:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177042.639159:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177042.639162:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177042.639164:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177042.639166:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177042.639168:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177042.639171:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:1.0:1786177042.639175:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177042.639176:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177042.639178:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314380. 00000100:00000001:1.0:1786177042.639182:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177042.639183:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.639185:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177042.639189:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316300 x1872930235568128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.639196:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177042.639197:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177042.639199:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177042.639201:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177042.639202:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177042.639203:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177042.639205:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177042.639206:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:1.0:1786177042.639207:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177042.639208:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177042.639209:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316300. 00000100:00000001:1.0:1786177042.639211:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177042.639212:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.639213:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177042.639216:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317800 x1872930235568256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.639221:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177042.639221:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177042.639222:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177042.639224:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177042.639225:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177042.639226:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177042.639227:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177042.639229:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:1.0:1786177042.639232:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177042.639233:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177042.639234:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317800. 00000100:00000001:1.0:1786177042.639237:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177042.639238:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.639240:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177042.639243:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316a00 x1872930235568384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.639249:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177042.639251:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177042.639252:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:1.0:1786177042.639255:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177042.639256:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177042.639258:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177042.639260:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177042.639261:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:1.0:1786177042.639264:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177042.639265:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177042.639267:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316a00. 00000100:00000001:1.0:1786177042.639269:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177042.639270:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.639272:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177042.639275:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09315c00 x1872930235568512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177067 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177042.639283:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177042.639284:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177042.639286:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177042.639288:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177042.639289:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177042.639290:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177042.639292:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177042.639293:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:1.0:1786177042.639296:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177042.639297:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177042.639298:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09315c00. 00000100:00000001:1.0:1786177042.639300:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177042.639302:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.639304:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177042.639311:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177042.639313:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177042.639316:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177042.639318:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177043.148462:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177043.148472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177043.148474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177043.148482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177043.148486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177043.148490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177043.148494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177043.148497:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177043.148501:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177043.532462:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177043.532469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177043.532470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177043.532474:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177043.532474:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177043.532481:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177043.532482:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177043.532485:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177043.532486:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177043.532489:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177043.532491:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177043.532492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177043.532495:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177043.532498:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177043.532500:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177043.532548:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177043.532552:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177043.532554:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.0:1786177044.108469:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:1.0:1786177044.108477:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:1.0:1786177044.108482:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:1.0:1786177044.108625:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:0.0:1786177044.108633:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1786177044.108639:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1786177044.108644:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1786177044.108647:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177044.172470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177044.172479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177044.172482:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177044.172489:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177044.172495:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177044.172499:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177044.172502:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177044.172505:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177044.172509:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177044.556463:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177044.556466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177044.556471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177044.556472:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177044.556476:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177044.556477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177044.556480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177044.556484:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177044.556484:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177044.556486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177044.556492:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177044.556497:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177044.556500:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177044.556503:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177044.556507:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177044.556548:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177044.556552:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177044.556556:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177045.196529:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177045.196550:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177045.196553:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177045.196560:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177045.196564:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177045.196568:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177045.196573:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177045.196576:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177045.196581:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177045.580527:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177045.580537:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177045.580545:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177045.580546:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177045.580550:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177045.580552:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177045.580559:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177045.580565:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177045.580566:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177045.580571:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177045.580575:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177045.580579:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177045.580582:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177045.580584:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177045.580586:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177045.580623:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177045.580628:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177045.580631:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177046.220466:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177046.220473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177046.220476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177046.220481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177046.220485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177046.220489:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177046.220492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177046.220494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177046.220496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177046.540477:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306393345, at 4306393408 00000100:00000040:1.0:1786177046.540477:0:206554:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306393345, at 4306393408 00000100:00000010:1.0:1786177046.540491:0:206554:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62b40. 00000100:00000010:0.0:1786177046.540493:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62660. 00000100:00000001:1.0:1786177046.604467:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177046.604473:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177046.604475:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177046.604479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177046.604480:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177046.604489:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177046.604489:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177046.604492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177046.604496:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177046.604498:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177046.604502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177046.604502:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177046.604507:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177046.604510:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177046.604514:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177046.604560:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177046.604563:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177046.604566:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177047.244471:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177047.244478:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177047.244480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177047.244485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177047.244490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177047.244493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177047.244496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177047.244498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177047.244500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.628460:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177047.628467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177047.628467:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177047.628469:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177047.628472:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177047.628474:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177047.628479:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177047.628480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177047.628482:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177047.628484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.628485:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177047.628490:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177047.628492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177047.628493:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177047.628496:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177047.628557:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177047.628562:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177047.628566:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177047.756482:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177047.756494:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177047.756496:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177047.756499:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177047.756503:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177047.756509:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11722 00000400:00000200:3.0:1786177047.756517:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177047.756524:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177047.756527:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177047.756530:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756534:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177047.756536:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756539:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177047.756543:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177047.756546:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177047.756549:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177047.756552:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177047.756555:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177047.756558:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177047.756562:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756564:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177047.756566:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756568:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177047.756571:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177047.756574:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756576:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177047.756578:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177047.756581:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177047.756583:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177047.756584:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177047.756586:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177047.756596:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786177047.756600:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177047.756604:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177047.756606:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177047.756609:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177047.756612:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177047.756616:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177047.756624:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786177047.756631:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177047.756642:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177047.756651:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177047.756654:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177047.756658:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177047.756680:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177047.756682:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177047.756685:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177047.756686:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177047.756687:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177047.756689:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177047.756691:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11722 00000400:00000200:3.0:1786177047.756694:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177047.756696:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177047.756698:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177047.756700:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756702:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177047.756704:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756706:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177047.756708:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177047.756709:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177047.756711:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177047.756713:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177047.756714:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177047.756716:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1786177047.756717:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177047.756722:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1786177047.756722:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177047.756725:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177047.756726:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177047.756730:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756731:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177047.756735:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000040:1.0:1786177047.756736:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177047.756739:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756744:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177047.756748:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177047.756752:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1786177047.756753:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177047.756754:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177047.756757:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.756759:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1786177047.756760:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1786177047.756764:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1786177047.756765:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1786177047.756768:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1786177047.756769:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177047.756772:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177047.756774:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235568640:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177047.756793:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177047.756797:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:3.0:1786177047.756798:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 02000000:00000001:1.0:1786177047.756799:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.756803:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1786177047.756804:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177047.756807:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177047.756808:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177047.756809:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad800. 00000100:00000001:3.0:1786177047.756810:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177047.756811:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177047.756813:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.756817:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177047.756818:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786177047.756823:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177047.756826:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177047.756831:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00100000:3.0:1786177047.756831:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177047.756839:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000001:3.0:1786177047.756840:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177047.756844:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177047.756847:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235568640, portal 25 00000100:00000040:3.0:1786177047.756848:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.756851:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177047.756854:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280811776 : -110371428739840 : ffff9b9e26325500) 00000100:00000001:3.0:1786177047.756861:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177047.756863:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177047.756865:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177047.756866:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177047.756869:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1786177047.756870:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177047.756872:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177047.756879:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177047.756882:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177047.756884:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177047.756886:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235568640, offset 0 00000400:00000200:1.0:1786177047.756890:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177047.756897:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177047.756901:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177047.756905:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1786177047.756910:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11722 00000400:00000200:1.0:1786177047.756912:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177047.756919:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1786177047.756919:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177047.756923:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177047.756929:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426509:426509:256:4294967295] 192.168.202.132@tcp LPNI seq info [426471:426471:8:4294967295] 00000400:00000200:3.0:1786177047.756929:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177047.756936:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177047.756937:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177047.756944:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756948:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177047.756954:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177047.756959:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:3.0:1786177047.756960:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177047.756965:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177047.756968:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177047.756971:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:1.0:1786177047.756973:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177047.756974:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177047.756978:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177047.756979:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000010:1.0:1786177047.756981:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000040:3.0:1786177047.756983:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756987:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177047.756988:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.756989:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177047.756993:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000200:1.0:1786177047.756994:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786177047.756997:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177047.757001:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177047.757002:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 02000000:00000001:3.0:1786177047.757003:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177047.757007:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177047.757011:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786177047.757011:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177047.757013:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177047.757016:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177047.757018:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177047.757019:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177047.757022:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000800:00000200:1.1:1786177047.757023:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000020:00000040:3.0:1786177047.757026:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000800:00000100:1.0:1786177047.757030:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 02000000:00000001:3.0:1786177047.757031:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177047.757034:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177047.757035:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00000001:3.0:1786177047.757036:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177047.757038:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177047.757040:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177047.757041:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:3.0:1786177047.757044:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000400:00000200:1.0:1786177047.757048:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1786177047.757050:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177047.757055:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000100:00100000:3.0:1786177047.757056:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177047.757061:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:3.0:1786177047.757067:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177047.757069:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786177047.757070:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177047.757080:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177047.757080:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177047.757104:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177047.757108:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177047.757111:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1786177047.757111:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177047.757115:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177047.757117:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1786177047.757121:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177047.757121:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177047.757126:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11722 00000400:00000200:3.0:1786177047.757133:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177047.757139:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177047.757140:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177047.757141:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177047.757143:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1786177047.757146:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.757148:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177047.757151:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177047.757151:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.757153:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177047.757156:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1786177047.757158:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:3.0:1786177047.757158:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177047.757160:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177047.757162:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306398464 00000020:00000040:3.0:1786177047.757163:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177047.757167:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177047.757168:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000010:1.0:1786177047.757169:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62b40. 00000100:00000040:3.0:1786177047.757172:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.757174:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177047.757176:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177047.757177:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1786177047.757178:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177047.757182:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177047.757184:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177047.757186:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306398464 00000100:00000001:3.0:1786177047.757186:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177047.757187:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:1.0:1786177047.757191:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62060. 00000100:00000001:1.0:1786177047.757194:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177047.757196:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177047.757200:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177047.757203:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.757207:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177047.757208:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177047.757211:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000008:00100000:3.0:1786177047.757213:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1786177047.757217:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177047/real 1786177047] req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1786177047.757218:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177047.757224:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177047.757231:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000400:00000001:1.0:1786177047.757232:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177047.757235:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177047.757237:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000020:00000040:3.0:1786177047.757237:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000400:00000200:1.0:1786177047.757241:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 02000000:00000001:3.0:1786177047.757243:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1786177047.757244:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177047.757248:0:2392:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1786177047.757248:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177047.757253:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177047.757253:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177047.757256:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177047.757259:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177047.757264:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000200:1.0:1786177047.757266:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177047.757270:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.757276:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177047.757278:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1786177047.757279:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177047.757281:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177047.757283:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177047.757291:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177047.757292:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177047.757295:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177047.757302:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177047.757308:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177047.757313:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177047.757314:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177047.757324:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177047.757324:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177047.757330:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177047.757333:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177047.757338:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177047.757340:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11722 00000400:00000200:3.0:1786177047.757343:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177047.757348:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:3.0:1786177047.757348:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177047.757353:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177047.757354:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.757386:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177047.757387:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.757390:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177047.757391:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1786177047.757394:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00080000:1.0:1786177047.757395:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1786177047.757400:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1786177047.757403:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000040:3.0:1786177047.757405:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177047.757407:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1786177047.757408:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177047.757412:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:1.0:1786177047.757413:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177047.757414:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177047.757417:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177047.757420:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177047.757425:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177047.757428:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00100000:1.0:1786177047.757430:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235568640:192.168.202.132@tcp:250:kworker.0 00000100:00000040:3.0:1786177047.757433:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177047.757437:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177047.757437:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177047.757441:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786177047.757441:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177047.757445:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:1.0:1786177047.757446:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325500 x1872930235568640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177047.757452:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:1.0:1786177047.757454:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177047.757457:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:3.0:1786177047.757459:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000010:1.0:1786177047.757460:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:1.0:1786177047.757466:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1786177047.757466:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177047.757470:0:2392:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:3.0:1786177047.757471:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000020:00000040:1.0:1786177047.757474:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177047.757476:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1786177047.757479:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:3.0:1786177047.757481:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177047.757483:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786177047.757485:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1786177047.757489:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000008:00100000:3.0:1786177047.757490:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 02000000:00000001:1.0:1786177047.757492:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000008:00000001:3.0:1786177047.757494:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177047.757496:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325500. 00000100:00000001:3.0:1786177047.757499:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.757501:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177047.757504:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1786177047.757506:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000100:00000001:1.0:1786177047.757508:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1786177047.757509:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177047.757513:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177047.757514:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177047.757516:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177047.757517:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1786177047.757518:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177047.757521:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177047.757522:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.757527:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177047.757530:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:3.0:1786177047.757531:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:1.0:1786177047.757532:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177047.757537:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177047.757540:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.757556:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177047.757558:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.757562:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1786177047.757562:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177047.757565:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177047.757571:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177047.757579:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568768:192.168.202.132@tcp:38:kworker.0 00000100:00000001:3.0:1786177047.757580:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177047.757584:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.757588:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1786177047.757590:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177047.757592:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177047.757595:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.757598:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177047.757603:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad800. 00000100:00000001:3.0:1786177047.757605:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.757607:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177047.757612:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11732) 00000400:00000200:1.0:1786177047.757623:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177047.757628:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177047.757633:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177047.757637:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235568768, portal 10 00000100:00000001:1.0:1786177047.757639:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177047.757640:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280812672 : -110371428738944 : ffff9b9e26325880) 00000100:00000040:1.0:1786177047.757648:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.757656:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177047.757657:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177047.757659:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177047.757661:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235568768, offset 0 00000400:00000200:1.0:1786177047.757666:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177047.757671:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177047.757675:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177047.757678:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177047.757681:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177047.757683:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177047.757686:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177047.757689:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426510:426510:256:4294967295] 192.168.202.132@tcp LPNI seq info [426472:426472:8:4294967295] 00000400:00000200:1.0:1786177047.757694:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177047.757701:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177047.757706:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177047.757709:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177047.757712:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177047.757714:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786177047.757717:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177047.757719:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1786177047.757723:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177047.757724:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177047.757727:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177047.757729:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177047.757731:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177047.757733:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177047.757735:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177047.757737:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.757743:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177047.757745:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.757750:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.757751:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177047.757753:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177047.757755:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177047.757758:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177047.757759:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306394628 00000100:00000001:1.0:1786177047.757761:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306398464 00000100:00000001:1.0:1786177047.757763:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306394628 00000100:00000001:1.0:1786177047.757764:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306398464 00000100:00000001:1.0:1786177047.757766:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177047.757794:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.757795:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177047.757798:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177047/real 1786177047] req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177047.757805:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177047.757806:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177047.757807:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177047.757809:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177047.757810:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177047.757811:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177047.757814:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177047.757819:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.757823:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177047.757824:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.757825:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177047.757826:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177047.757828:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.757833:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177047.757835:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.757840:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177047.757841:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.757842:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177047.757843:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.757844:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177047.757846:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177047.757848:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177047.757850:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177047.757852:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177047.757854:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177047.757859:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568768:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177047.757862:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177047.757864:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.757867:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177047.757868:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.757870:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177047.757871:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177047.757872:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177047.757874:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568896:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177047.757876:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177047.757878:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177047.757879:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.757880:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177047.757882:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177047.757884:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177047.757886:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177047.757888:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177047.757890:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177047.757892:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235568896, portal 10 00000100:00000001:1.0:1786177047.757893:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177047.757894:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280821632 : -110371428729984 : ffff9b9e26327b80) 00000100:00000040:1.0:1786177047.757898:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.757903:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177047.757905:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177047.757906:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177047.757907:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235568896, offset 0 00000400:00000200:1.0:1786177047.757910:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177047.757913:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177047.757916:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177047.757918:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177047.757920:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177047.757921:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177047.757923:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177047.757925:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426511:426511:256:4294967295] 192.168.202.132@tcp LPNI seq info [426473:426473:8:4294967295] 00000400:00000200:1.0:1786177047.757928:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177047.757932:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177047.757936:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177047.757938:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177047.757940:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177047.757942:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786177047.757945:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177047.757947:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1786177047.757951:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177047.757953:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177047.757956:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177047.757960:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177047.757963:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177047.757964:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177047.757967:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177047.757970:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.757979:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177047.757982:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.757989:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.757991:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177047.757995:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177047.757998:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177047.758001:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177047.758003:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306394628 00000100:00000001:1.0:1786177047.758006:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306398464 00000100:00000001:1.0:1786177047.758008:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306394628 00000100:00000001:1.0:1786177047.758010:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306398464 00000100:00000001:1.0:1786177047.758013:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177047.758014:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.758016:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177047.758019:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177047/real 1786177047] req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177047.758029:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177047.758030:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177047.758032:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177047.758033:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177047.758035:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177047.758037:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177047.758039:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177047.758044:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758048:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177047.758049:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758050:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177047.758051:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177047.758053:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758058:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177047.758061:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758065:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177047.758067:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758068:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177047.758069:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758070:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177047.758071:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177047.758073:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177047.758075:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177047.758076:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177047.758078:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177047.758083:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235568896:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177047.758086:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177047.758088:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758092:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177047.758093:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.758094:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177047.758096:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177047.758097:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177047.758099:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569024:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177047.758101:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177047.758102:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177047.758103:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.758104:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177047.758107:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177047.758108:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177047.758122:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177047.758124:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177047.758127:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177047.758129:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235569024, portal 4 00000100:00000001:1.0:1786177047.758130:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177047.758131:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280815360 : -110371428736256 : ffff9b9e26326300) 00000100:00000040:1.0:1786177047.758135:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758140:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177047.758141:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177047.758143:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177047.758144:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235569024, offset 0 00000400:00000200:1.0:1786177047.758147:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177047.758149:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177047.758152:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177047.758155:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177047.758156:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177047.758158:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177047.758160:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177047.758162:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426512:426512:256:4294967295] 192.168.202.132@tcp LPNI seq info [426474:426474:8:4294967295] 00000400:00000200:1.0:1786177047.758165:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177047.758169:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177047.758172:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177047.758174:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177047.758177:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177047.758179:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786177047.758182:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177047.758184:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1786177047.758186:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177047.758188:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177047.758190:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177047.758192:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177047.758194:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177047.758196:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177047.758198:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177047.758200:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758205:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177047.758207:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758212:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.758213:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177047.758215:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177047.758216:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177047.758218:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177047.758219:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306394628 00000100:00000001:1.0:1786177047.758221:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306398464 00000100:00000001:1.0:1786177047.758223:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306394628 00000100:00000001:1.0:1786177047.758224:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306398464 00000100:00000001:1.0:1786177047.758226:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177047.758227:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.758228:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177047.758230:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177047/real 1786177047] req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177047.758235:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177047.758236:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177047.758237:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177047.758238:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177047.758239:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177047.758240:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177047.758242:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177047.758247:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758252:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177047.758253:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758255:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177047.758256:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177047.758258:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758262:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177047.758264:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758268:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177047.758270:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758270:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177047.758271:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758272:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177047.758273:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177047.758275:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177047.758277:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177047.758278:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177047.758280:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177047.758285:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569024:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177047.758288:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758290:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177047.758292:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325880 x1872930235568768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758296:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177047.758298:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177047.758300:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:1.0:1786177047.758303:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177047.758305:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177047.758306:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177047.758309:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177047.758310:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa400. 02000000:00000001:1.0:1786177047.758313:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177047.758314:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177047.758316:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325880. 00000100:00000001:1.0:1786177047.758319:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177047.758320:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758322:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177047.758326:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327b80 x1872930235568896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758334:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177047.758335:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177047.758337:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177047.758339:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177047.758341:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177047.758342:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177047.758344:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177047.758346:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab800. 02000000:00000001:1.0:1786177047.758348:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177047.758350:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177047.758351:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327b80. 00000100:00000001:1.0:1786177047.758353:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177047.758354:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758370:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177047.758374:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326300 x1872930235569024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758379:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177047.758380:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177047.758381:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177047.758383:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177047.758384:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177047.758384:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177047.758386:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177047.758387:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8c00. 02000000:00000001:1.0:1786177047.758388:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177047.758389:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177047.758390:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326300. 00000100:00000001:1.0:1786177047.758392:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177047.758392:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758394:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758403:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177047.758404:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.758405:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177047.758408:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177047.758409:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177047.758412:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758417:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177047.758418:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.758419:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177047.758420:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177047.758422:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177047.758423:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569152:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177047.758425:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177047.758427:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177047.758428:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177047.758429:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177047.758431:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177047.758432:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177047.758435:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177047.758436:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177047.758439:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177047.758441:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235569152, portal 4 00000100:00000001:1.0:1786177047.758442:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177047.758443:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280813568 : -110371428738048 : ffff9b9e26325c00) 00000100:00000040:1.0:1786177047.758447:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758455:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177047.758457:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177047.758458:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177047.758460:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235569152, offset 0 00000400:00000200:1.0:1786177047.758463:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177047.758465:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177047.758468:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177047.758470:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177047.758471:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177047.758473:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177047.758475:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177047.758477:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426513:426513:256:4294967295] 192.168.202.132@tcp LPNI seq info [426475:426475:8:4294967295] 00000400:00000200:1.0:1786177047.758480:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177047.758484:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177047.758487:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177047.758489:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177047.758491:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177047.758493:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786177047.758496:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177047.758498:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1786177047.758501:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177047.758502:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177047.758504:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177047.758506:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177047.758508:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177047.758510:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177047.758511:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177047.758513:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758519:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177047.758521:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177072 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758529:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.758530:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177047.758532:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177047.758534:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177047.758536:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177047.758537:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306394631 00000100:00000001:1.0:1786177047.758539:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306398464 00000100:00000001:1.0:1786177047.758540:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306394631 00000100:00000001:1.0:1786177047.758542:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306398464 00000100:00000001:1.0:1786177047.758543:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177047.758545:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.758546:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177047.758548:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177047/real 1786177047] req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177047.758553:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177047.758553:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177047.758554:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177047.758556:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177047.758557:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177047.758558:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177047.758560:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177047.758565:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758569:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177047.758570:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758571:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177047.758572:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177047.758574:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758579:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177047.758581:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758586:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177047.758587:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758588:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177047.758589:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758590:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177047.758591:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177047.758592:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177047.758594:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177047.758595:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177047.758598:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177047.758603:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569152:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177047.758606:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758608:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177047.758612:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325c00 x1872930235569152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177072 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177047.758618:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177047.758619:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177047.758621:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177047.758625:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177047.758627:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177047.758629:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177047.758631:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177047.758633:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8400. 02000000:00000001:1.0:1786177047.758636:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177047.758639:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177047.758641:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325c00. 00000100:00000001:1.0:1786177047.758644:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177047.758646:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758648:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177047.758651:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177047.758653:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177047.758655:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177047.758657:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177048.268464:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177048.268472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177048.268474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177048.268480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177048.268484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177048.268487:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177048.268490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177048.268493:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177048.268497:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177048.652473:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177048.652477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177048.652480:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177048.652481:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177048.652483:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177048.652488:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177048.652489:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177048.652492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177048.652493:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177048.652496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177048.652498:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177048.652498:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177048.652504:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177048.652507:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177048.652510:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177048.652560:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177048.652565:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177048.652569:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177049.292499:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177049.292511:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177049.292515:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177049.292522:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177049.292528:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177049.292533:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177049.292536:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177049.292541:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177049.292545:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177049.676460:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177049.676467:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177049.676468:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177049.676469:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177049.676474:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177049.676476:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177049.676481:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177049.676484:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177049.676485:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177049.676489:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177049.676490:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177049.676491:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177049.676495:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177049.676496:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177049.676501:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177049.676505:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177049.676507:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177049.676509:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177049.676511:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177049.676514:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177049.676515:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177049.676578:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177049.676582:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177049.676585:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177050.316450:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177050.316457:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177050.316459:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177050.316464:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177050.316468:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177050.316471:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177050.316473:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177050.316475:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177050.316477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177050.700449:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177050.700456:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177050.700458:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177050.700458:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177050.700462:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177050.700464:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177050.700466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177050.700468:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177050.700469:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177050.700472:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177050.700474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177050.700475:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177050.700477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177050.700479:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177050.700483:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177050.700535:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177050.700538:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177050.700541:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177051.340604:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177051.340611:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177051.340613:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177051.340617:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177051.340626:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177051.340629:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177051.340631:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177051.340633:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177051.340636:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177051.660432:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306398464, at 4306398528 00000100:00000040:1.0:1786177051.660442:0:206554:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306398464, at 4306398528 00000100:00000010:0.0:1786177051.660446:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62b40. 00000100:00000010:1.0:1786177051.660455:0:206554:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62060. 00000100:00000001:1.0:1786177051.724486:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177051.724489:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177051.724495:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177051.724496:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177051.724498:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177051.724500:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177051.724502:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177051.724507:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:1.0:1786177051.724509:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177051.724510:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177051.724515:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177051.724517:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177051.724518:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177051.724522:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177051.724523:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177051.724526:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177051.724528:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177051.724530:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177051.724532:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177051.724533:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177051.724535:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177051.724538:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177051.724539:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177051.724540:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177051.724542:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177051.724545:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177051.724547:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177051.724614:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177051.724618:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177051.724622:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177052.364458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177052.364465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177052.364467:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177052.364472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177052.364476:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177052.364481:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177052.364484:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177052.364488:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177052.364491:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.748429:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177052.748436:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177052.748438:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177052.748440:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177052.748443:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177052.748444:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.748448:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177052.748450:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177052.748452:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.748452:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.748456:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177052.748458:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177052.748460:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177052.748461:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.748465:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177052.748518:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177052.748523:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177052.748527:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177052.876488:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177052.876499:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177052.876502:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177052.876506:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177052.876510:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177052.876517:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11727 00000400:00000200:3.0:1786177052.876526:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177052.876533:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177052.876536:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177052.876540:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177052.876543:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177052.876547:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177052.876549:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177052.876554:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177052.876556:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177052.876559:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177052.876562:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177052.876565:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177052.876568:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177052.876571:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177052.876573:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177052.876575:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177052.876578:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177052.876581:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177052.876584:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177052.876587:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177052.876590:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177052.876593:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177052.876596:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177052.876599:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177052.876601:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177052.876611:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786177052.876615:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177052.876619:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177052.876621:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177052.876624:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177052.876627:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177052.876631:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177052.876638:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177052.876645:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177052.876654:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177052.876667:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177052.876672:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177052.876677:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177052.876701:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177052.876705:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177052.876709:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177052.876710:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177052.876712:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177052.876715:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177052.876719:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11727 00000400:00000200:3.0:1786177052.876724:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177052.876758:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177052.876762:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1786177052.876762:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177052.876767:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1786177052.876769:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177052.876771:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177052.876776:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177052.876780:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177052.876783:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177052.876790:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177052.876796:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1786177052.876796:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177052.876800:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1786177052.876801:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177052.876805:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1786177052.876806:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177052.876809:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1786177052.876809:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 02000000:00000001:1.0:1786177052.876815:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177052.876816:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00100000:1.0:1786177052.876820:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235569280:192.168.202.132@tcp:250:kworker.0 00000020:00000001:3.0:1786177052.876822:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177052.876826:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177052.876830:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1786177052.876831:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177052.876832:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.876838:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1786177052.876838:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:1.0:1786177052.876844:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 00000100:00000040:3.0:1786177052.876844:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177052.876849:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177052.876849:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177052.876855:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000400:00000200:1.0:1786177052.876858:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177052.876864:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00080000:3.0:1786177052.876864:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000010:1.0:1786177052.876871:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000001:3.0:1786177052.876873:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177052.876877:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:1.0:1786177052.876879:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235569280, portal 25 00000100:00000001:1.0:1786177052.876882:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1786177052.876883:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177052.876884:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280810880 : -110371428740736 : ffff9b9e26325180) 02000000:00000001:3.0:1786177052.876891:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177052.876895:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177052.876897:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177052.876903:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1786177052.876904:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177052.876909:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000008:00000001:3.0:1786177052.876912:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177052.876913:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177052.876918:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235569280, offset 0 00000100:00000001:3.0:1786177052.876918:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177052.876927:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000400:00000200:1.0:1786177052.876928:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177052.876934:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177052.876938:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1786177052.876939:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:3.0:1786177052.876942:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177052.876947:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:3.0:1786177052.876948:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177052.876951:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1786177052.876953:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1786177052.876955:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177052.876959:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 02000000:00000010:3.0:1786177052.876963:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786177052.876968:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177052.876973:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177052.876978:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00100000:3.0:1786177052.876980:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177052.876983:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426514:426514:256:4294967295] 192.168.202.132@tcp LPNI seq info [426476:426476:8:4294967295] 00000400:00000200:1.0:1786177052.876996:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177052.877004:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177052.877006:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:3.0:1786177052.877010:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177052.877015:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177052.877019:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1786177052.877021:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177052.877032:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177052.877037:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1786177052.877040:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786177052.877045:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786177052.877047:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1786177052.877048:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00080000:3.0:1786177052.877055:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1786177052.877057:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177052.877059:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000010:1.0:1786177052.877061:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00080000:3.0:1786177052.877063:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1786177052.877066:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177052.877070:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177052.877074:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177052.877075:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:3.0:1786177052.877078:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177052.877080:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177052.877087:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177052.877087:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11727 00000400:00000200:3.0:1786177052.877100:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177052.877103:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1786177052.877108:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000040:1.0:1786177052.877109:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177052.877116:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177052.877119:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.877121:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1786177052.877124:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177052.877127:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177052.877129:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177052.877134:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177052.877135:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177052.877140:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306403584 00000100:00000001:3.0:1786177052.877146:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000010:1.0:1786177052.877147:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62060. 00000020:00000001:3.0:1786177052.877154:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177052.877156:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000100:00000001:1.0:1786177052.877158:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1786177052.877159:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177052.877163:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177052.877167:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000001:1.0:1786177052.877170:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306403584 00000020:00000001:3.0:1786177052.877171:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000010:1.0:1786177052.877177:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62900. 00000100:00000040:3.0:1786177052.877178:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177052.877184:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1786177052.877184:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177052.877189:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177052.877192:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1786177052.877194:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177052.877197:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1786177052.877200:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00000001:1.0:1786177052.877202:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1786177052.877208:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00100000:1.0:1786177052.877212:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177052/real 1786177052] req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177052.877215:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177052.877220:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177052.877226:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1786177052.877227:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177052.877229:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177052.877233:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 02000000:00000001:3.0:1786177052.877233:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177052.877238:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000100:00000001:3.0:1786177052.877240:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:1.0:1786177052.877241:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000008:00100000:3.0:1786177052.877244:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1786177052.877245:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177052.877250:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1786177052.877252:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177052.877257:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177052.877265:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1786177052.877265:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786177052.877272:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177052.877279:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177052.877283:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.877284:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177052.877288:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177052.877289:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177052.877290:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177052.877292:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177052.877295:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.877299:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177052.877302:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.877307:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177052.877309:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.877310:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177052.877311:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.877312:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177052.877314:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177052.877315:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177052.877318:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177052.877319:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177052.877322:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177052.877327:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235569280:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177052.877330:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.877332:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177052.877334:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325180 x1872930235569280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177052.877337:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177052.877341:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1786177052.877342:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.877344:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177052.877349:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177052.877352:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000010:3.0:1786177052.877352:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000020:00000001:1.0:1786177052.877355:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177052.877382:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177052.877386:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177052.877389:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177052.877391:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.877394:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177052.877399:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:3.0:1786177052.877401:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:1.0:1786177052.877402:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325180. 00000100:00000001:1.0:1786177052.877409:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177052.877412:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177052.877417:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177052.877418:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177052.877423:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.877428:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1786177052.877430:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.877432:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.877439:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177052.877442:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177052.877445:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177052.877451:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177052.877452:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177052.877457:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177052.877468:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1786177052.877469:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177052.877471:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177052.877474:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1786177052.877475:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177052.877480:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1786177052.877480:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1786177052.877484:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177052.877488:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:1.0:1786177052.877489:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569408:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177052.877494:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1786177052.877495:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11727 02000000:00000001:1.0:1786177052.877497:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177052.877499:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.877502:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177052.877504:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 00000400:00000200:3.0:1786177052.877506:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177052.877507:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177052.877514:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177052.877514:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000010:1.0:1786177052.877518:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00080000:3.0:1786177052.877521:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1786177052.877523:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177052.877529:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235569408, portal 10 00000100:00000040:3.0:1786177052.877530:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177052.877532:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177052.877534:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818048 : -110371428733568 : ffff9b9e26326d80) 00000100:00000001:3.0:1786177052.877536:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177052.877542:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177052.877543:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177052.877548:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177052.877554:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1786177052.877556:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:1.0:1786177052.877559:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177052.877561:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000400:00000010:1.0:1786177052.877563:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000020:00000001:3.0:1786177052.877566:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:1.0:1786177052.877567:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235569408, offset 0 00000020:00000040:3.0:1786177052.877571:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000400:00000200:1.0:1786177052.877572:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000020:00000001:3.0:1786177052.877577:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000200:1.0:1786177052.877581:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:3.0:1786177052.877583:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000200:1.0:1786177052.877587:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177052.877592:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1786177052.877593:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177052.877596:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:3.0:1786177052.877600:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177052.877601:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177052.877607:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000040:3.0:1786177052.877607:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177052.877613:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426515:426515:256:4294967295] 192.168.202.132@tcp LPNI seq info [426477:426477:8:4294967295] 00000100:00000001:3.0:1786177052.877614:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177052.877621:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:3.0:1786177052.877621:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177052.877629:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177052.877636:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177052.877638:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 02000000:00000001:3.0:1786177052.877640:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177052.877646:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1786177052.877649:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 02000000:00000001:3.0:1786177052.877651:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177052.877655:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000010:1.0:1786177052.877656:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000008:00100000:3.0:1786177052.877659:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000800:00000200:1.0:1786177052.877664:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000008:00000001:3.0:1786177052.877666:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177052.877671:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1786177052.877671:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177052.877678:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000800:00000200:1.1:1786177052.877679:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177052.877685:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000020:00000040:3.0:1786177052.877686:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000800:00000100:1.0:1786177052.877690:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 02000000:00000001:3.0:1786177052.877692:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000010:1.0:1786177052.877693:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 02000000:00000001:3.0:1786177052.877697:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177052.877699:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177052.877705:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1786177052.877705:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177052.877710:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1786177052.877711:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177052.877714:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 02000000:00000001:3.0:1786177052.877715:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.877717:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177052.877722:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1786177052.877723:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786177052.877747:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.877751:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177052.877759:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786177052.877759:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177052.877778:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.877781:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177052.877787:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177052.877787:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177052.877793:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177052.877794:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177052.877801:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177052.877803:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306399748 00000100:00000001:1.0:1786177052.877806:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306403584 00000100:00000040:3.0:1786177052.877806:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.877811:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306399748 00000100:00000001:1.0:1786177052.877814:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306403584 00000100:00000001:1.0:1786177052.877817:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177052.877821:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.877825:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177052.877830:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177052/real 1786177052] req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177052.877832:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177052.877838:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:1.0:1786177052.877842:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177052.877845:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1786177052.877847:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000010:1.0:1786177052.877848:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177052.877852:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000100:00000001:3.0:1786177052.877852:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:1.0:1786177052.877856:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00080000:3.0:1786177052.877857:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177052.877859:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177052.877865:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177052.877865:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177052.877872:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11727 00000100:00000200:1.0:1786177052.877879:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1786177052.877885:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177052.877889:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177052.877892:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.877895:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:3.0:1786177052.877895:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177052.877898:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177052.877903:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1786177052.877904:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177052.877911:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177052.877913:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177052.877918:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177052.877920:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177052.877926:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177052.877930:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177052.877931:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177052.877933:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.877938:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1786177052.877939:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177052.877940:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.877943:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000040:3.0:1786177052.877943:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000001:1.0:1786177052.877946:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1786177052.877948:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00080000:1.0:1786177052.877950:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000040:3.0:1786177052.877952:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00080000:1.0:1786177052.877956:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000020:00000001:3.0:1786177052.877958:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177052.877959:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:3.0:1786177052.877965:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:1.0:1786177052.877966:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177052.877976:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00100000:1.0:1786177052.877981:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569408:192.168.202.132@tcp:38:kworker.0 00000100:00000001:3.0:1786177052.877982:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177052.877986:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.877990:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786177052.877990:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177052.877995:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326d80 x1872930235569408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177052.877996:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177052.878004:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:1.0:1786177052.878008:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177052.878010:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177052.878013:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 00000100:00080000:3.0:1786177052.878013:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:1.0:1786177052.878018:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177052.878020:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:3.0:1786177052.878020:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177052.878023:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177052.878025:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:1.0:1786177052.878027:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177052.878030:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:3.0:1786177052.878031:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177052.878032:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177052.878035:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177052.878038:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326d80. 02000000:00000001:3.0:1786177052.878038:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.878043:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177052.878045:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177052.878045:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177052.878050:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1786177052.878050:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177052.878053:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177052.878055:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1786177052.878057:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.878058:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177052.878062:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177052.878063:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.878064:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:3.0:1786177052.878070:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000100:00000040:1.0:1786177052.878071:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177052.878079:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177052.878080:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177052.878083:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.878086:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1786177052.878086:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177052.878091:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177052.878092:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.878093:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177052.878098:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1786177052.878099:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569536:192.168.202.132@tcp:38:kworker.0 02000000:00000001:3.0:1786177052.878102:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177052.878103:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177052.878107:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1786177052.878108:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.878109:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.878114:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177052.878117:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000010:3.0:1786177052.878117:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 02000000:00000001:1.0:1786177052.878122:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177052.878126:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177052.878126:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177052.878130:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177052.878136:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00100000:3.0:1786177052.878138:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1786177052.878141:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235569536, portal 10 00000100:00000001:1.0:1786177052.878148:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177052.878151:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280820736 : -110371428730880 : ffff9b9e26327800) 00000100:00000001:3.0:1786177052.878158:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177052.878161:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177052.878164:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177052.878170:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878185:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177052.878191:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177052.878193:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177052.878194:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177052.878199:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235569536, offset 0 00000100:00000040:3.0:1786177052.878201:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11737) 00000400:00000200:1.0:1786177052.878204:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177052.878210:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177052.878214:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177052.878216:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177052.878218:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177052.878220:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177052.878222:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177052.878226:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426516:426516:256:4294967295] 192.168.202.132@tcp LPNI seq info [426478:426478:8:4294967295] 00000400:00000200:1.0:1786177052.878230:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177052.878234:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177052.878237:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177052.878240:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177052.878243:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177052.878245:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786177052.878248:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177052.878249:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1786177052.878252:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177052.878254:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177052.878256:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177052.878258:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177052.878260:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177052.878262:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177052.878263:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177052.878266:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878272:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177052.878274:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878283:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.878284:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177052.878286:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177052.878288:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177052.878290:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177052.878292:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306399748 00000100:00000001:1.0:1786177052.878294:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306403584 00000100:00000001:1.0:1786177052.878295:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306399748 00000100:00000001:1.0:1786177052.878297:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306403584 00000100:00000001:1.0:1786177052.878298:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177052.878299:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.878301:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177052.878303:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177052/real 1786177052] req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177052.878313:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177052.878314:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177052.878315:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177052.878317:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177052.878318:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177052.878319:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177052.878322:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177052.878327:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878331:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177052.878332:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878333:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177052.878334:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177052.878336:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878341:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177052.878344:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878348:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177052.878350:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878351:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177052.878351:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878353:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177052.878354:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177052.878371:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177052.878376:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177052.878378:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177052.878382:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177052.878390:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569536:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177052.878394:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177052.878398:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878405:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177052.878406:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.878409:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177052.878411:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177052.878413:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177052.878416:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569664:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177052.878420:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177052.878421:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177052.878423:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.878424:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177052.878427:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad800. 02000000:00000001:1.0:1786177052.878428:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177052.878431:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177052.878433:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177052.878435:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177052.878437:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235569664, portal 4 00000100:00000001:1.0:1786177052.878439:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177052.878440:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280814464 : -110371428737152 : ffff9b9e26325f80) 00000100:00000040:1.0:1786177052.878444:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878449:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177052.878450:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177052.878451:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177052.878453:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235569664, offset 0 00000400:00000200:1.0:1786177052.878455:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177052.878458:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177052.878460:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177052.878462:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177052.878464:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177052.878466:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177052.878468:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177052.878470:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426517:426517:256:4294967295] 192.168.202.132@tcp LPNI seq info [426479:426479:8:4294967295] 00000400:00000200:1.0:1786177052.878473:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177052.878477:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177052.878480:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177052.878482:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177052.878485:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177052.878487:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786177052.878489:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177052.878491:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1786177052.878494:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177052.878496:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177052.878498:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177052.878500:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177052.878502:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177052.878503:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177052.878504:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177052.878507:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878512:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177052.878514:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878518:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.878519:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177052.878521:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177052.878523:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177052.878525:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177052.878526:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306399749 00000100:00000001:1.0:1786177052.878528:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306403584 00000100:00000001:1.0:1786177052.878529:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306399749 00000100:00000001:1.0:1786177052.878531:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306403584 00000100:00000001:1.0:1786177052.878532:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177052.878533:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.878535:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177052.878537:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177052/real 1786177052] req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177052.878541:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177052.878542:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177052.878543:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177052.878544:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177052.878545:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177052.878547:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177052.878549:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177052.878555:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878558:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177052.878559:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878560:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177052.878561:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177052.878563:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878567:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177052.878569:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878573:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177052.878575:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878576:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177052.878577:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878578:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177052.878579:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177052.878580:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177052.878582:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177052.878584:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177052.878586:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177052.878591:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569664:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177052.878593:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878595:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177052.878597:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327800 x1872930235569536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878602:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177052.878603:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177052.878604:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177052.878605:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177052.878606:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177052.878607:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177052.878608:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177052.878610:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:1.0:1786177052.878611:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177052.878613:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177052.878614:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327800. 00000100:00000001:1.0:1786177052.878617:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177052.878618:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878620:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177052.878623:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325f80 x1872930235569664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878629:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177052.878630:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177052.878632:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:1.0:1786177052.878634:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177052.878636:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177052.878637:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177052.878639:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177052.878641:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:1.0:1786177052.878643:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177052.878644:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177052.878646:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325f80. 00000100:00000001:1.0:1786177052.878648:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177052.878649:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878652:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878655:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177052.878657:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.878659:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177052.878660:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177052.878662:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177052.878666:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878672:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177052.878674:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.878675:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177052.878676:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177052.878678:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177052.878680:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569792:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177052.878682:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177052.878683:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177052.878685:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177052.878686:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177052.878688:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad800. 02000000:00000001:1.0:1786177052.878689:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177052.878692:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177052.878693:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177052.878695:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177052.878697:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235569792, portal 4 00000100:00000001:1.0:1786177052.878699:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177052.878700:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280819840 : -110371428731776 : ffff9b9e26327480) 00000100:00000040:1.0:1786177052.878704:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878708:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177052.878710:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177052.878711:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177052.878712:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235569792, offset 0 00000400:00000200:1.0:1786177052.878715:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177052.878717:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177052.878720:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177052.878722:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177052.878724:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177052.878742:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177052.878745:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177052.878748:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426518:426518:256:4294967295] 192.168.202.132@tcp LPNI seq info [426480:426480:8:4294967295] 00000400:00000200:1.0:1786177052.878751:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177052.878755:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177052.878758:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177052.878761:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177052.878764:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177052.878766:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786177052.878768:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177052.878770:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1786177052.878773:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177052.878774:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177052.878776:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177052.878779:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177052.878781:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177052.878782:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177052.878784:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177052.878786:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878791:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177052.878794:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177077 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878798:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.878799:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177052.878801:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177052.878803:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177052.878805:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177052.878807:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306399751 00000100:00000001:1.0:1786177052.878808:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306403584 00000100:00000001:1.0:1786177052.878810:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306399751 00000100:00000001:1.0:1786177052.878811:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306403584 00000100:00000001:1.0:1786177052.878813:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177052.878814:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.878816:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177052.878818:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177052/real 1786177052] req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177052.878823:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177052.878824:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177052.878825:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177052.878826:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177052.878827:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177052.878829:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177052.878831:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177052.878836:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878840:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177052.878841:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878842:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177052.878843:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177052.878845:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878849:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177052.878851:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878855:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177052.878857:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878858:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177052.878858:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878859:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177052.878860:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177052.878863:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177052.878865:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177052.878867:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177052.878869:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177052.878874:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235569792:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177052.878876:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878877:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177052.878879:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327480 x1872930235569792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177077 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177052.878883:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177052.878884:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177052.878885:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:1.0:1786177052.878886:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177052.878887:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177052.878888:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177052.878889:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177052.878890:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 02000000:00000001:1.0:1786177052.878892:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177052.878893:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177052.878894:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327480. 00000100:00000001:1.0:1786177052.878896:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177052.878896:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878898:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177052.878900:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177052.878901:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177052.878902:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177052.878904:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177053.388454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177053.388461:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177053.388463:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177053.388469:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177053.388472:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177053.388476:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177053.388479:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177053.388481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177053.388484:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177053.772475:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177053.772476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177053.772482:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177053.772483:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177053.772486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177053.772488:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177053.772489:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177053.772495:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177053.772496:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177053.772497:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177053.772503:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177053.772508:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177053.772511:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177053.772514:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177053.772517:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177053.772557:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177053.772562:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177053.772565:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177054.412440:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177054.412447:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177054.412448:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177054.412453:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177054.412457:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177054.412460:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177054.412463:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177054.412465:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177054.412467:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177054.796477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177054.796479:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177054.796484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177054.796486:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177054.796489:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177054.796490:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177054.796492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177054.796497:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177054.796498:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177054.796498:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177054.796504:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177054.796508:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177054.796511:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177054.796514:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177054.796518:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177054.796561:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177054.796566:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177054.796569:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177055.436469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177055.436477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177055.436481:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177055.436487:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177055.436491:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177055.436496:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177055.436499:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177055.436502:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177055.436506:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177055.820454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177055.820462:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177055.820463:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177055.820466:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177055.820467:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177055.820474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177055.820475:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177055.820476:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177055.820483:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177055.820484:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177055.820487:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177055.820489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177055.820494:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177055.820496:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177055.820500:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177055.820536:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177055.820539:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177055.820542:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177056.460444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177056.460454:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177056.460457:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177056.460463:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177056.460466:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177056.460469:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177056.460471:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177056.460474:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177056.460477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177056.780425:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306403584, at 4306403648 00000100:00000040:1.0:1786177056.780425:0:206554:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306403584, at 4306403648 00000100:00000010:0.0:1786177056.780440:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62060. 00000100:00000010:1.0:1786177056.780442:0:206554:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62900. 00000100:00000001:1.0:1786177056.844482:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177056.844485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177056.844491:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177056.844491:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177056.844495:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177056.844499:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177056.844501:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177056.844503:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177056.844508:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177056.844510:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177056.844513:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177056.844514:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177056.844519:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177056.844524:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177056.844528:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177056.844583:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177056.844588:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177056.844591:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177057.484763:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177057.484769:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177057.484770:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177057.484774:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177057.484777:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177057.484780:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177057.484782:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177057.484784:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177057.484786:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.868477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177057.868485:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177057.868488:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177057.868494:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177057.868496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177057.868499:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177057.868502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.868505:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177057.868509:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177057.868511:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177057.868513:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.868516:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177057.868519:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177057.868522:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177057.868523:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177057.868593:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177057.868608:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177057.868612:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177057.996469:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177057.996481:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177057.996484:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177057.996487:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177057.996491:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177057.996497:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11732 00000400:00000200:2.0:1786177057.996503:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177057.996508:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177057.996511:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177057.996513:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996516:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177057.996518:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996519:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177057.996522:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177057.996524:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177057.996526:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177057.996529:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177057.996530:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177057.996532:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177057.996534:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996535:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177057.996542:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996543:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177057.996545:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177057.996548:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996550:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177057.996552:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177057.996555:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177057.996557:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.996559:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177057.996561:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177057.996568:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735180. 00000020:00000040:2.0:1786177057.996571:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177057.996574:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177057.996575:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.996578:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177057.996580:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177057.996583:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177057.996589:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9000. 00000100:00000001:2.0:1786177057.996595:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177057.996601:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177057.996609:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177057.996613:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177057.996616:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ba0f078] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177057.996631:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177057.996633:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177057.996636:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177057.996637:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177057.996638:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177057.996640:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177057.996642:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11732 00000400:00000200:2.0:1786177057.996644:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177057.996646:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177057.996647:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177057.996649:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996651:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177057.996652:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996653:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177057.996655:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177057.996656:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177057.996657:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177057.996659:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177057.996660:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177057.996662:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177057.996664:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996666:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177057.996668:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996670:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177057.996671:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177057.996674:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996687:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.996689:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1786177057.996689:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177057.996695:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1786177057.996695:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177057.996700:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1786177057.996700:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.996705:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177057.996708:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177057.996711:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177057.996712:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177057.996713:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177057.996718:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734e00. 00000020:00000040:2.0:1786177057.996723:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177057.996728:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177057.996730:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.996735:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177057.996735:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177057.996738:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177057.996740:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177057.996741:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177057.996748:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:1.0:1786177057.996749:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1786177057.996752:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177057.996754:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1786177057.996760:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177057.996761:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177057.996771:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235569920:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177057.996772:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177057.996776:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.996781:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1786177057.996783:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001f62212] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177057.996787:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177057.996793:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.996793:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177057.996796:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1786177057.996798:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1786177057.996800:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177057.996803:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177057.996805:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:1.0:1786177057.996808:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 00000100:00000001:2.0:1786177057.996808:0:40394:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1786177057.996813:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177057.996815:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11732 00000400:00000200:1.0:1786177057.996822:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177057.996824:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177057.996831:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000200:2.0:1786177057.996831:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177057.996838:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1786177057.996841:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000040:2.0:1786177057.996842:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996848:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177057.996851:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:1.0:1786177057.996852:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235569920, portal 25 00000100:00000001:2.0:1786177057.996855:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177057.996857:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177057.996861:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620096 : -110371239931520 : ffff9b9e31735180) 00000020:00000001:2.0:1786177057.996862:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177057.996864:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177057.996866:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177057.996869:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177057.996872:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177057.996873:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177057.996874:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177057.996881:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996885:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177057.996890:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996895:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177057.996897:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:2.0:1786177057.996898:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177057.996904:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000040:1.0:1786177057.996908:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177057.996909:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177057.996913:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1786177057.996914:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 02000000:00000001:2.0:1786177057.996917:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1786177057.996921:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235569920, offset 0 02000000:00000001:2.0:1786177057.996922:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.996928:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177057.996932:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177057.996937:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000008:00000001:2.0:1786177057.996938:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.996942:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177057.996949:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000010:2.0:1786177057.996950:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000020:00000040:2.0:1786177057.996955:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177057.996959:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 02000000:00000001:2.0:1786177057.996960:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177057.996964:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.996967:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1786177057.996969:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1786177057.996970:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177057.996974:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177057.996977:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 02000000:00000010:2.0:1786177057.996979:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aabc00. 00000400:00000200:1.0:1786177057.996984:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1786177057.996984:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177057.996990:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00100000:2.0:1786177057.996994:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177057.997001:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426519:426519:256:4294967295] 192.168.202.132@tcp LPNI seq info [426481:426481:8:4294967295] 00000100:00000001:2.0:1786177057.997008:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177057.997013:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177057.997013:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177057.997021:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177057.997034:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177057.997036:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177057.997041:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177057.997045:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177057.997048:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.0:1786177057.997049:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00080000:2.0:1786177057.997050:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177057.997056:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177057.997059:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11732 00000800:00000010:1.0:1786177057.997060:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000400:00000200:2.0:1786177057.997065:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177057.997069:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177057.997072:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1786177057.997075:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:2.0:1786177057.997077:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.997081:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1786177057.997083:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000100:00000040:2.0:1786177057.997084:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.997087:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177057.997090:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177057.997093:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177057.997097:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:1.1:1786177057.997098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:2.0:1786177057.997099:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177057.997103:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177057.997104:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177057.997107:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.997109:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177057.997110:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.997113:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:1.1:1786177057.997117:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000020:00000040:2.0:1786177057.997118:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000800:00000100:1.0:1786177057.997128:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177057.997132:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177057.997139:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1786177057.997139:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177057.997146:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177057.997148:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1786177057.997150:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177057.997156:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1786177057.997157:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 02000000:00000001:2.0:1786177057.997160:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177057.997163:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:2.0:1786177057.997164:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177057.997167:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177057.997171:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:2.0:1786177057.997171:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.997176:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177057.997180:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000100:00000200:1.0:1786177057.997181:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177057.997183:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177057.997189:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177057.997191:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.997194:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177057.997197:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177057.997199:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.997204:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:2.0:1786177057.997204:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:2.0:1786177057.997207:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177057.997213:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177057.997216:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177057.997247:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177057.997248:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.997250:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.997253:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1786177057.997257:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997262:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.997265:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177057.997268:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177057.997272:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:2.0:1786177057.997273:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177057.997278:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177057.997279:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306408704 00000100:00080000:2.0:1786177057.997280:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177057.997286:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177057.997288:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11732 00000100:00000010:1.0:1786177057.997290:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62900. 00000400:00000200:2.0:1786177057.997295:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177057.997298:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177057.997300:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177057.997301:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000040:2.0:1786177057.997304:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.997307:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177057.997308:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.997310:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177057.997312:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177057.997313:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306408704 00000020:00000040:2.0:1786177057.997314:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177057.997318:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000010:1.0:1786177057.997321:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62ba0. 00000020:00000040:2.0:1786177057.997321:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177057.997323:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177057.997328:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177057.997330:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000040:2.0:1786177057.997331:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.997335:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177057.997338:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1786177057.997338:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177057.997343:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177057.997346:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177057.997346:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177057.997352:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177057.997354:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1786177057.997354:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177057.997381:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177057.997383:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177057.997385:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.997387:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177057.997388:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177057.997390:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.997392:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177057.997395:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000020:00000040:2.0:1786177057.997398:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177057.997402:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1786177057.997403:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177057/real 1786177057] req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177057.997403:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177057.997408:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177057.997411:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177057.997415:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177057.997422:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000400:00000001:1.0:1786177057.997423:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177057.997428:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1786177057.997429:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177057.997431:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177057.997436:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000100:00100000:2.0:1786177057.997439:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177057.997440:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177057.997445:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177057.997453:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177057.997457:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177057.997463:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177057.997472:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177057.997474:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997491:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177057.997496:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177057.997503:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11742) 00000100:00000001:1.0:1786177057.997507:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.997512:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177057.997514:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177057.997521:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997529:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177057.997534:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997547:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177057.997550:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.997552:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177057.997553:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.997555:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177057.997558:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177057.997561:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177057.997564:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177057.997567:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177057.997571:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177057.997579:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235569920:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177057.997585:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.997587:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177057.997590:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735180 x1872930235569920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997597:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177057.997599:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177057.997602:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177057.997605:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177057.997607:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177057.997608:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177057.997611:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177057.997613:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:1.0:1786177057.997616:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177057.997618:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177057.997620:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735180. 00000100:00000001:1.0:1786177057.997623:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177057.997624:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.997626:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.997633:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177057.997636:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.997638:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177057.997639:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177057.997641:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177057.997645:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997652:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177057.997653:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177057.997656:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177057.997658:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177057.997660:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177057.997662:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570048:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177057.997666:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177057.997667:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177057.997669:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177057.997671:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177057.997674:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177057.997690:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177057.997694:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177057.997697:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177057.997701:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177057.997704:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235570048, portal 10 00000100:00000001:1.0:1786177057.997706:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177057.997707:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469619200 : -110371239932416 : ffff9b9e31734e00) 00000100:00000040:1.0:1786177057.997714:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997720:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177057.997722:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177057.997725:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177057.997727:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235570048, offset 0 00000400:00000200:1.0:1786177057.997731:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177057.997735:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177057.997739:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177057.997743:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177057.997745:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177057.997747:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177057.997750:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177057.997753:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426520:426520:256:4294967295] 192.168.202.132@tcp LPNI seq info [426482:426482:8:4294967295] 00000400:00000200:1.0:1786177057.997757:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177057.997764:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177057.997768:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177057.997772:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177057.997776:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177057.997779:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786177057.997783:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177057.997786:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1786177057.997790:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177057.997792:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177057.997795:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177057.997798:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177057.997802:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177057.997803:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177057.997806:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177057.997810:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997818:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177057.997821:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997827:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.997829:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177057.997832:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177057.997835:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177057.997839:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177057.997841:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306404868 00000100:00000001:1.0:1786177057.997844:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306408704 00000100:00000001:1.0:1786177057.997846:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306404868 00000100:00000001:1.0:1786177057.997848:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306408704 00000100:00000001:1.0:1786177057.997850:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177057.997852:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.997854:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177057.997858:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177057/real 1786177057] req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177057.997866:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177057.997867:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177057.997869:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177057.997871:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177057.997873:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177057.997875:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177057.997878:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177057.997886:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997892:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177057.997893:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.997895:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177057.997897:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177057.997901:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997907:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177057.997911:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997918:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177057.997920:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.997922:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177057.997923:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.997925:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177057.997927:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177057.997930:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177057.997933:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177057.997935:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177057.997939:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177057.997947:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570048:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177057.997951:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177057.997954:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.997960:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177057.997962:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177057.997965:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177057.997966:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177057.997968:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177057.997971:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570176:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177057.997974:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177057.997976:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177057.997978:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177057.997980:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177057.997983:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177057.997985:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177057.997988:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177057.997991:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177057.997994:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177057.997996:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235570176, portal 10 00000100:00000001:1.0:1786177057.997998:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177057.998000:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469618304 : -110371239933312 : ffff9b9e31734a80) 00000100:00000040:1.0:1786177057.998006:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998013:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177057.998015:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177057.998017:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177057.998020:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235570176, offset 0 00000400:00000200:1.0:1786177057.998024:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177057.998027:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177057.998031:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177057.998034:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177057.998037:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177057.998039:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177057.998041:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177057.998044:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426521:426521:256:4294967295] 192.168.202.132@tcp LPNI seq info [426483:426483:8:4294967295] 00000400:00000200:1.0:1786177057.998048:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177057.998054:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177057.998059:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177057.998062:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177057.998066:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177057.998069:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786177057.998072:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177057.998075:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1786177057.998079:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177057.998081:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177057.998084:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177057.998087:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177057.998090:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177057.998092:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177057.998094:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177057.998097:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998110:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177057.998114:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998121:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.998123:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177057.998126:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177057.998129:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177057.998132:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177057.998134:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306404868 00000100:00000001:1.0:1786177057.998137:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306408704 00000100:00000001:1.0:1786177057.998139:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306404868 00000100:00000001:1.0:1786177057.998141:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306408704 00000100:00000001:1.0:1786177057.998143:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177057.998146:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.998147:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177057.998151:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177057/real 1786177057] req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177057.998158:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177057.998160:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177057.998161:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177057.998164:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177057.998166:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177057.998168:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177057.998172:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177057.998180:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998188:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177057.998189:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998191:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177057.998193:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177057.998196:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998203:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177057.998207:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998214:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177057.998217:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998219:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177057.998220:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998222:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177057.998224:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177057.998226:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177057.998229:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177057.998232:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177057.998236:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177057.998244:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570176:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177057.998247:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177057.998251:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998257:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177057.998259:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177057.998261:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177057.998263:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177057.998264:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177057.998267:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570304:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177057.998270:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177057.998272:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177057.998274:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177057.998275:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177057.998278:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261adc00. 02000000:00000001:1.0:1786177057.998280:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177057.998283:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177057.998286:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177057.998289:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177057.998292:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235570304, portal 4 00000100:00000001:1.0:1786177057.998294:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177057.998295:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469628160 : -110371239923456 : ffff9b9e31737100) 00000100:00000040:1.0:1786177057.998301:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998307:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177057.998309:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177057.998311:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177057.998313:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235570304, offset 0 00000400:00000200:1.0:1786177057.998317:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177057.998321:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177057.998325:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177057.998328:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177057.998331:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177057.998334:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177057.998336:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177057.998339:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426522:426522:256:4294967295] 192.168.202.132@tcp LPNI seq info [426484:426484:8:4294967295] 00000400:00000200:1.0:1786177057.998344:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177057.998350:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177057.998354:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177057.998377:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177057.998382:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177057.998385:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786177057.998389:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177057.998392:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1786177057.998396:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177057.998399:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177057.998401:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177057.998405:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177057.998408:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177057.998410:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177057.998412:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177057.998416:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998423:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177057.998426:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998432:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.998434:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177057.998437:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177057.998440:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177057.998443:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177057.998445:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306404869 00000100:00000001:1.0:1786177057.998447:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306408704 00000100:00000001:1.0:1786177057.998449:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306404869 00000100:00000001:1.0:1786177057.998451:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306408704 00000100:00000001:1.0:1786177057.998453:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177057.998455:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.998457:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177057.998461:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177057/real 1786177057] req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177057.998467:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177057.998469:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177057.998471:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177057.998473:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177057.998474:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177057.998476:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177057.998479:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177057.998487:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998493:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177057.998495:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998496:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177057.998498:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177057.998501:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998508:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177057.998512:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998518:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177057.998520:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998523:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177057.998524:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998526:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177057.998529:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177057.998532:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177057.998535:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177057.998538:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177057.998543:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177057.998557:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570304:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177057.998561:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177057.998564:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998570:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177057.998571:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177057.998574:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177057.998577:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177057.998579:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177057.998583:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570432:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177057.998586:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177057.998588:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177057.998590:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177057.998591:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177057.998595:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177057.998597:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177057.998600:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177057.998602:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177057.998605:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177057.998608:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235570432, portal 4 00000100:00000001:1.0:1786177057.998610:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177057.998611:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469623680 : -110371239927936 : ffff9b9e31735f80) 00000100:00000040:1.0:1786177057.998617:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998623:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177057.998625:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177057.998627:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177057.998629:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235570432, offset 0 00000400:00000200:1.0:1786177057.998633:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177057.998637:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177057.998640:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177057.998643:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177057.998645:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177057.998648:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177057.998650:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177057.998653:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426523:426523:256:4294967295] 192.168.202.132@tcp LPNI seq info [426485:426485:8:4294967295] 00000400:00000200:1.0:1786177057.998657:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177057.998662:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177057.998666:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177057.998670:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177057.998674:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177057.998696:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786177057.998701:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177057.998704:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1786177057.998708:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177057.998711:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177057.998714:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177057.998718:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177057.998721:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177057.998723:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177057.998725:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177057.998730:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998738:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177057.998742:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177082 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998749:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.998750:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177057.998753:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177057.998757:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177057.998761:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177057.998763:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306404871 00000100:00000001:1.0:1786177057.998766:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306408704 00000100:00000001:1.0:1786177057.998769:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306404871 00000100:00000001:1.0:1786177057.998771:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306408704 00000100:00000001:1.0:1786177057.998773:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177057.998775:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.998777:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177057.998782:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177057/real 1786177057] req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177057.998790:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177057.998791:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177057.998793:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177057.998795:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177057.998797:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177057.998799:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177057.998802:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177057.998810:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998816:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177057.998818:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998820:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177057.998821:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177057.998825:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998831:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177057.998835:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998841:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177057.998844:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998846:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177057.998847:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998849:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177057.998850:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177057.998853:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177057.998856:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177057.998858:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177057.998862:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177057.998869:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570432:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177057.998873:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998875:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177057.998878:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734e00 x1872930235570048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998884:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177057.998886:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177057.998888:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177057.998891:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177057.998893:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177057.998894:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177057.998896:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177057.998898:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:1.0:1786177057.998901:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177057.998902:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177057.998904:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734e00. 00000100:00000001:1.0:1786177057.998906:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177057.998908:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998909:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177057.998913:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734a80 x1872930235570176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998925:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177057.998926:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177057.998928:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177057.998930:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177057.998931:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177057.998932:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177057.998934:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177057.998935:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:1.0:1786177057.998938:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177057.998939:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177057.998940:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734a80. 00000100:00000001:1.0:1786177057.998942:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177057.998943:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998944:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177057.998948:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737100 x1872930235570304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998954:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177057.998955:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177057.998956:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261adc00. 02000000:00000001:1.0:1786177057.998958:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177057.998959:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177057.998961:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177057.998963:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177057.998964:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:1.0:1786177057.998966:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177057.998967:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177057.998968:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737100. 00000100:00000001:1.0:1786177057.998970:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177057.998972:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.998973:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177057.998976:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735f80 x1872930235570432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177082 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177057.998982:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177057.998983:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177057.998985:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177057.998987:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177057.998989:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177057.998990:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177057.998992:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177057.998993:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:1.0:1786177057.998996:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177057.998997:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177057.998998:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735f80. 00000100:00000001:1.0:1786177057.999000:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177057.999001:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.999003:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177057.999008:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177057.999010:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177057.999012:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177057.999013:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177058.508517:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177058.508526:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177058.508528:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177058.508534:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177058.508538:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177058.508543:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177058.508547:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177058.508550:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177058.508554:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177058.892466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177058.892472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177058.892472:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177058.892474:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177058.892478:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177058.892480:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177058.892484:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177058.892485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177058.892487:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177058.892489:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177058.892491:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177058.892494:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177058.892496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177058.892497:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177058.892500:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177058.892566:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177058.892571:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177058.892575:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177059.532486:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177059.532496:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177059.532499:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177059.532505:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177059.532510:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177059.532514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177059.532517:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177059.532520:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177059.532523:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177059.916437:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177059.916445:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177059.916452:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177059.916455:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177059.916461:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177059.916463:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177059.916466:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177059.916469:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1786177059.916469:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177059.916473:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177059.916475:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177059.916476:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177059.916479:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177059.916480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177059.916481:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177059.916487:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177059.916487:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177059.916492:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177059.916494:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177059.916497:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177059.916498:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177059.916557:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177059.916561:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177059.916564:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177060.556463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177060.556472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177060.556475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177060.556481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177060.556485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177060.556488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177060.556491:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177060.556495:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177060.556499:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177060.940479:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177060.940484:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177060.940488:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177060.940490:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177060.940492:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177060.940499:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177060.940500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177060.940503:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177060.940506:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177060.940510:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177060.940512:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177060.940513:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177060.940517:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177060.940521:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177060.940525:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177060.940576:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177060.940580:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177060.940583:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177061.580475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177061.580491:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177061.580494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177061.580501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177061.580504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177061.580507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177061.580510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177061.580513:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177061.580517:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177061.900465:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306408704, at 4306408768 00000100:00000040:1.0:1786177061.900465:0:206554:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306408704, at 4306408768 00000100:00000010:0.0:1786177061.900480:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62900. 00000100:00000010:1.0:1786177061.900481:0:206554:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ba0. 00000100:00000001:1.0:1786177061.964444:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177061.964448:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177061.964453:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177061.964455:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177061.964457:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177061.964460:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1786177061.964465:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177061.964466:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177061.964474:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177061.964474:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177061.964479:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177061.964481:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177061.964483:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177061.964487:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177061.964488:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177061.964492:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177061.964495:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177061.964498:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177061.964499:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177061.964506:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177061.964509:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177061.964510:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177061.964512:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177061.964514:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177061.964516:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177061.964517:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177061.964519:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177061.964590:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177061.964595:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177061.964598:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177062.604473:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177062.604482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177062.604485:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177062.604492:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177062.604497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177062.604507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177062.604510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177062.604512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177062.604515:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177062.988465:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177062.988468:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177062.988471:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177062.988472:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177062.988473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177062.988477:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177062.988478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177062.988479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177062.988481:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177062.988483:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177062.988484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177062.988485:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177062.988488:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177062.988491:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177062.988494:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177062.988543:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177062.988548:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177062.988552:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177063.116465:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177063.116476:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177063.116479:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177063.116483:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177063.116488:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177063.116496:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11737 00000400:00000200:3.0:1786177063.116504:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177063.116511:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177063.116515:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177063.116518:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177063.116522:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177063.116525:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177063.116528:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177063.116532:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177063.116535:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177063.116538:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177063.116541:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177063.116544:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177063.116547:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177063.116550:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177063.116552:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177063.116554:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177063.116556:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177063.116560:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177063.116564:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177063.116567:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177063.116569:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177063.116573:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177063.116576:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177063.116579:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177063.116581:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177063.116591:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786177063.116594:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177063.116599:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177063.116601:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177063.116604:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177063.116607:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177063.116611:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177063.116619:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786177063.116655:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177063.116676:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177063.116702:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177063.116707:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177063.116713:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177063.116742:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177063.116746:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177063.116750:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177063.116752:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177063.116753:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177063.116756:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177063.116759:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11737 00000400:00000200:3.0:1786177063.116764:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177063.116769:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177063.116772:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177063.116775:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177063.116777:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1786177063.116778:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177063.116783:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177063.116784:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177063.116787:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177063.116789:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1786177063.116794:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177063.116798:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000100:00000040:1.0:1786177063.116800:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177063.116801:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177063.116805:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177063.116811:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177063.116817:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177063.116820:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1786177063.116824:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177063.116825:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177063.116831:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1786177063.116831:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177063.116834:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1786177063.116835:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177063.116838:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177063.116840:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00100000:1.0:1786177063.116844:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235570560:192.168.202.132@tcp:250:kworker.0 00000020:00000040:3.0:1786177063.116846:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:1.0:1786177063.116850:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1786177063.116854:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:1.0:1786177063.116855:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177063.116860:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177063.116860:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177063.116866:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177063.116871:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177063.116872:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1786177063.116877:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177063.116878:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 00000100:00000001:3.0:1786177063.116880:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1786177063.116882:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1786177063.116887:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177063.116888:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177063.116893:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000008:00000001:3.0:1786177063.116894:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177063.116898:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177063.116900:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000010:3.0:1786177063.116904:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000100:00000200:1.0:1786177063.116907:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235570560, portal 25 00000020:00000040:3.0:1786177063.116909:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000100:00000001:1.0:1786177063.116911:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177063.116914:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280816256 : -110371428735360 : ffff9b9e26326680) 02000000:00000001:3.0:1786177063.116916:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177063.116920:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177063.116921:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177063.116925:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177063.116929:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177063.116932:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1786177063.116933:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177063.116937:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177063.116943:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 02000000:00000010:3.0:1786177063.116943:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000200:1.0:1786177063.116946:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235570560, offset 0 00000100:00000001:3.0:1786177063.116951:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177063.116952:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177063.116962:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:3.0:1786177063.116963:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177063.116969:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177063.116977:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177063.116982:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:3.0:1786177063.116983:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177063.116986:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1786177063.116989:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177063.116991:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177063.116998:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426524:426524:256:4294967295] 192.168.202.132@tcp LPNI seq info [426486:426486:8:4294967295] 00000100:00000040:3.0:1786177063.117001:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177063.117004:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177063.117017:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:3.0:1786177063.117023:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177063.117025:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177063.117031:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00080000:3.0:1786177063.117031:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177063.117038:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.0:1786177063.117041:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177063.117042:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.0:1786177063.117047:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00080000:3.0:1786177063.117047:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177063.117055:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1786177063.117056:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786177063.117062:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11737 00000800:00000200:1.1:1786177063.117063:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000400:00000200:3.0:1786177063.117070:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000100:1.0:1786177063.117074:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000400:00000200:3.0:1786177063.117078:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000010:1.0:1786177063.117080:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00080000:3.0:1786177063.117083:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177063.117089:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1786177063.117089:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177063.117095:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177063.117099:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1786177063.117101:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177063.117106:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177063.117108:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000020:00000001:3.0:1786177063.117109:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:1.0:1786177063.117112:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000020:00000040:3.0:1786177063.117113:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177063.117117:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177063.117118:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:3.0:1786177063.117122:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000100:00000200:1.0:1786177063.117125:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177063.117127:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177063.117135:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000001:1.0:1786177063.117139:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786177063.117143:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177063.117144:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177063.117148:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177063.117156:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177063.117156:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177063.117159:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1786177063.117163:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177063.117166:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1786177063.117168:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:1.0:1786177063.117172:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177063.117176:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306413824 00000100:00000010:1.0:1786177063.117181:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62ba0. 00000100:00080000:3.0:1786177063.117184:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177063.117186:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177063.117194:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306413824 00000100:00000010:1.0:1786177063.117197:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62c60. 00000100:00000001:1.0:1786177063.117200:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1786177063.117200:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.117204:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177063.117205:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177063.117208:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.117211:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1786177063.117211:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1786177063.117217:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177063/real 1786177063] req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177063.117217:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177063.117221:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177063.117225:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000001:1.0:1786177063.117229:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:3.0:1786177063.117230:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1786177063.117231:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1786177063.117234:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177063.117235:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177063.117238:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177063.117240:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000010:3.0:1786177063.117240:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000100:00000001:1.0:1786177063.117242:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177063.117247:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177063.117247:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177063.117253:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177063.117256:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177063.117261:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1786177063.117263:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177063.117264:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177063.117267:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.117274:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:3.0:1786177063.117274:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:1.0:1786177063.117276:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.117279:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1786177063.117280:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.117281:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177063.117289:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786177063.117291:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177063.117300:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177063.117306:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1786177063.117308:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177063.117310:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177063.117318:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177063.117324:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177063.117327:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177063.117330:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1786177063.117331:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177063.117334:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177063.117337:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177063.117337:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177063.117342:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1786177063.117343:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177063.117347:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177063.117347:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11737 00000100:00000001:1.0:1786177063.117353:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177063.117387:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177063.117391:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:3.0:1786177063.117391:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177063.117398:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000400:00000200:3.0:1786177063.117401:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177063.117402:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1786177063.117408:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177063.117409:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177063.117418:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00100000:1.0:1786177063.117422:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235570560:192.168.202.132@tcp:250:kworker.0 00000100:00000001:3.0:1786177063.117424:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177063.117431:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177063.117432:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.117437:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1786177063.117437:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:1.0:1786177063.117442:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326680 x1872930235570560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177063.117444:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177063.117448:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:1.0:1786177063.117452:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1786177063.117453:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1786177063.117455:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177063.117458:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 00000020:00000040:3.0:1786177063.117458:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 02000000:00000001:1.0:1786177063.117463:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1786177063.117465:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177063.117466:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177063.117468:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:3.0:1786177063.117471:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177063.117472:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177063.117475:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:1.0:1786177063.117479:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:3.0:1786177063.117481:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177063.117482:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177063.117484:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326680. 00000100:00000001:3.0:1786177063.117487:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177063.117488:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177063.117491:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.117494:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177063.117494:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177063.117498:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177063.117501:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:3.0:1786177063.117502:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:1.0:1786177063.117504:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.117508:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1786177063.117509:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177063.117510:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177063.117513:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177063.117517:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177063.117518:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177063.117523:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177063.117528:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177063.117529:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177063.117529:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177063.117533:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177063.117537:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177063.117538:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177063.117540:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177063.117543:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:1.0:1786177063.117544:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570688:192.168.202.132@tcp:38:kworker.0 00000008:00100000:3.0:1786177063.117547:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177063.117548:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177063.117550:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177063.117553:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1786177063.117553:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177063.117558:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1786177063.117559:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177063.117562:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177063.117564:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177063.117566:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000400:00000200:1.0:1786177063.117569:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:3.0:1786177063.117573:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000400:00000010:1.0:1786177063.117574:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 02000000:00000001:3.0:1786177063.117579:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1786177063.117580:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 02000000:00000001:3.0:1786177063.117582:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177063.117585:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235570688, portal 10 00000100:00000001:3.0:1786177063.117586:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177063.117589:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1786177063.117589:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177063.117592:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280807296 : -110371428744320 : ffff9b9e26324380) 02000000:00000001:3.0:1786177063.117592:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177063.117599:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000040:1.0:1786177063.117600:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177063.117605:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.117610:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177063.117616:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00100000:3.0:1786177063.117616:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177063.117620:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177063.117625:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235570688, offset 0 00000100:00000001:3.0:1786177063.117642:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177063.117647:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177063.117650:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177063.117654:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177063.117658:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177063.117668:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177063.117670:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00080000:3.0:1786177063.117672:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177063.117675:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1786177063.117677:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:1.0:1786177063.117679:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:3.0:1786177063.117681:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1786177063.117684:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1786177063.117685:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1786177063.117688:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:3.0:1786177063.117689:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177063.117694:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426525:426525:256:4294967295] 192.168.202.132@tcp LPNI seq info [426487:426487:8:4294967295] 00000100:00080000:3.0:1786177063.117696:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11737 00000400:00000200:1.0:1786177063.117701:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177063.117705:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177063.117713:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177063.117716:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00080000:3.0:1786177063.117717:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1786177063.117725:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177063.117725:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177063.117730:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:1.0:1786177063.117731:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000040:3.0:1786177063.117734:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:1.0:1786177063.117739:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177063.117739:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177063.117744:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:1.0:1786177063.117745:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000020:00000040:3.0:1786177063.117747:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177063.117750:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177063.117754:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000800:00000200:1.1:1786177063.117754:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177063.117758:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000020:00000001:3.0:1786177063.117759:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177063.117764:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000800:00000100:1.0:1786177063.117767:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177063.117770:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000010:1.0:1786177063.117771:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00000001:3.0:1786177063.117773:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177063.117778:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1786177063.117778:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177063.117780:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177063.117783:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000400:00000200:1.0:1786177063.117789:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1786177063.117789:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177063.117795:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177063.117797:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 02000000:00000001:3.0:1786177063.117797:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177063.117801:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1786177063.117802:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 02000000:00000001:3.0:1786177063.117806:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.117808:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786177063.117809:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177063.117813:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177063.117817:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177063.117818:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177063.117819:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177063.117826:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786177063.117830:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177063.117831:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1786177063.117832:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177063.117835:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177063.117837:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177063.117837:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177063.117840:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177063.117842:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177063.117848:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:1.0:1786177063.117849:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177063.117851:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.117852:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177063.117857:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177063.117860:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1786177063.117862:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177063.117870:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177063.117874:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177063.117875:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306409988 00000100:00000001:3.0:1786177063.117879:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.117880:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306413824 00000100:00000001:1.0:1786177063.117884:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306409988 00000100:00000001:1.0:1786177063.117888:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306413824 00000100:00000040:3.0:1786177063.117888:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.117893:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177063.117897:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.117901:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1786177063.117904:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177063.117908:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177063/real 1786177063] req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177063.117910:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11747) 00000400:00000001:1.0:1786177063.117918:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177063.117920:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177063.117921:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177063.117923:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177063.117924:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177063.117925:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177063.117928:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177063.117935:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.117939:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177063.117940:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.117941:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177063.117942:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177063.117944:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.117948:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177063.117951:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.117955:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177063.117957:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.117958:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177063.117958:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.117959:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177063.117961:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177063.117963:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177063.117965:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177063.117966:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177063.117969:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177063.117974:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570688:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177063.117977:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177063.117979:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.117987:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177063.117988:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177063.117990:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177063.117992:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177063.117993:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177063.117995:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570816:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177063.117997:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177063.117998:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177063.117999:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177063.118000:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177063.118003:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177063.118004:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177063.118006:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177063.118008:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177063.118010:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177063.118012:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235570816, portal 10 00000100:00000001:1.0:1786177063.118013:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177063.118014:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818944 : -110371428732672 : ffff9b9e26327100) 00000100:00000040:1.0:1786177063.118018:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118022:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177063.118024:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177063.118025:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177063.118027:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235570816, offset 0 00000400:00000200:1.0:1786177063.118030:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177063.118032:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177063.118035:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177063.118037:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177063.118038:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177063.118040:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177063.118042:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177063.118044:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426526:426526:256:4294967295] 192.168.202.132@tcp LPNI seq info [426488:426488:8:4294967295] 00000400:00000200:1.0:1786177063.118047:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177063.118052:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177063.118057:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177063.118061:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177063.118065:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177063.118068:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786177063.118072:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177063.118075:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1786177063.118079:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177063.118081:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177063.118083:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177063.118087:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177063.118091:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177063.118093:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177063.118095:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177063.118099:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118108:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177063.118110:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118115:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.118116:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177063.118118:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177063.118121:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177063.118123:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177063.118124:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306409988 00000100:00000001:1.0:1786177063.118126:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306413824 00000100:00000001:1.0:1786177063.118128:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306409988 00000100:00000001:1.0:1786177063.118129:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306413824 00000100:00000001:1.0:1786177063.118130:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177063.118133:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.118134:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177063.118136:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177063/real 1786177063] req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177063.118142:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177063.118143:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177063.118144:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177063.118145:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177063.118146:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177063.118148:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177063.118150:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177063.118155:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118159:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177063.118160:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118161:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177063.118162:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177063.118164:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118168:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177063.118170:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118174:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177063.118176:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118177:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177063.118178:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118179:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177063.118181:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177063.118182:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177063.118185:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177063.118186:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177063.118189:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177063.118194:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570816:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177063.118196:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118197:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177063.118199:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324380 x1872930235570688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118203:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177063.118204:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177063.118205:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177063.118207:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177063.118208:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177063.118209:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177063.118210:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177063.118212:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9400. 02000000:00000001:1.0:1786177063.118213:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177063.118215:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177063.118216:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324380. 00000100:00000001:1.0:1786177063.118217:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177063.118218:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118219:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177063.118221:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327100 x1872930235570816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118226:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177063.118226:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177063.118227:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177063.118228:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177063.118229:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177063.118230:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177063.118231:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177063.118232:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9000. 02000000:00000001:1.0:1786177063.118233:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177063.118234:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177063.118235:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327100. 00000100:00000001:1.0:1786177063.118237:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177063.118238:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118239:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118241:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177063.118242:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.118243:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177063.118244:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177063.118245:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177063.118248:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118252:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177063.118253:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177063.118255:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177063.118256:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177063.118257:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177063.118259:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570944:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177063.118261:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177063.118262:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177063.118263:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177063.118265:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177063.118267:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177063.118268:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177063.118270:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177063.118271:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177063.118274:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177063.118276:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235570944, portal 4 00000100:00000001:1.0:1786177063.118277:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177063.118278:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280816256 : -110371428735360 : ffff9b9e26326680) 00000100:00000040:1.0:1786177063.118282:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118290:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177063.118292:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177063.118293:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177063.118295:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235570944, offset 0 00000400:00000200:1.0:1786177063.118297:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177063.118300:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177063.118302:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177063.118304:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177063.118305:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177063.118307:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177063.118309:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177063.118311:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426527:426527:256:4294967295] 192.168.202.132@tcp LPNI seq info [426489:426489:8:4294967295] 00000400:00000200:1.0:1786177063.118314:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177063.118319:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177063.118324:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177063.118327:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177063.118331:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177063.118334:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786177063.118339:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177063.118342:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1786177063.118345:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177063.118348:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177063.118351:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177063.118355:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177063.118377:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177063.118379:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177063.118381:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177063.118384:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118390:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177063.118392:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118401:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.118402:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177063.118404:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177063.118407:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177063.118409:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177063.118410:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306409989 00000100:00000001:1.0:1786177063.118412:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306413824 00000100:00000001:1.0:1786177063.118414:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306409989 00000100:00000001:1.0:1786177063.118415:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306413824 00000100:00000001:1.0:1786177063.118416:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177063.118418:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.118419:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177063.118421:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177063/real 1786177063] req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177063.118426:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177063.118427:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177063.118428:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177063.118430:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177063.118431:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177063.118432:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177063.118434:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177063.118440:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118443:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177063.118444:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118445:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177063.118446:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177063.118448:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118453:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177063.118456:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118460:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177063.118461:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118462:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177063.118463:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118464:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177063.118465:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177063.118467:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177063.118469:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177063.118470:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177063.118472:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177063.118477:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235570944:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177063.118479:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177063.118481:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118485:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177063.118486:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177063.118487:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177063.118489:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177063.118490:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177063.118492:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571072:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177063.118494:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177063.118495:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177063.118497:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177063.118498:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177063.118500:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177063.118501:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177063.118503:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177063.118505:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177063.118507:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177063.118509:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235571072, portal 4 00000100:00000001:1.0:1786177063.118510:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177063.118511:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280819840 : -110371428731776 : ffff9b9e26327480) 00000100:00000040:1.0:1786177063.118515:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118520:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177063.118521:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177063.118522:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177063.118524:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235571072, offset 0 00000400:00000200:1.0:1786177063.118526:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177063.118529:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177063.118531:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177063.118533:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177063.118534:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177063.118536:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177063.118537:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177063.118540:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426528:426528:256:4294967295] 192.168.202.132@tcp LPNI seq info [426490:426490:8:4294967295] 00000400:00000200:1.0:1786177063.118542:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177063.118547:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177063.118551:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177063.118555:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177063.118559:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177063.118562:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786177063.118566:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177063.118569:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1786177063.118573:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177063.118575:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177063.118578:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177063.118581:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177063.118584:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177063.118586:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177063.118589:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177063.118593:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118601:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177063.118604:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177088 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118611:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.118613:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177063.118616:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177063.118619:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177063.118623:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177063.118625:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306409991 00000100:00000001:1.0:1786177063.118639:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306413824 00000100:00000001:1.0:1786177063.118641:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306409991 00000100:00000001:1.0:1786177063.118644:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306413824 00000100:00000001:1.0:1786177063.118646:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177063.118647:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.118649:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177063.118652:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177063/real 1786177063] req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177063.118658:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177063.118659:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177063.118660:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177063.118662:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177063.118663:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177063.118665:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177063.118667:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177063.118673:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118677:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177063.118678:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118679:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177063.118680:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177063.118682:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118687:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177063.118689:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118694:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177063.118695:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118696:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177063.118697:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118698:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177063.118699:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177063.118701:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177063.118702:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177063.118704:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177063.118706:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177063.118711:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571072:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177063.118713:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118714:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177063.118717:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326680 x1872930235570944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118721:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177063.118723:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177063.118724:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177063.118725:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177063.118726:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177063.118727:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177063.118728:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177063.118730:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa800. 02000000:00000001:1.0:1786177063.118732:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177063.118733:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177063.118734:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326680. 00000100:00000001:1.0:1786177063.118735:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177063.118736:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118737:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177063.118739:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327480 x1872930235571072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177088 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177063.118743:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177063.118744:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177063.118745:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177063.118746:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177063.118747:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177063.118748:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177063.118749:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177063.118749:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa000. 02000000:00000001:1.0:1786177063.118750:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177063.118751:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177063.118752:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327480. 00000100:00000001:1.0:1786177063.118753:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177063.118754:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118755:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177063.118757:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177063.118758:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177063.118759:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177063.118760:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177063.628481:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177063.628489:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177063.628491:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177063.628496:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177063.628501:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177063.628505:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177063.628507:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177063.628510:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177063.628512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177064.012453:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177064.012454:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177064.012461:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177064.012462:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177064.012466:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177064.012470:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177064.012471:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177064.012475:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177064.012476:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177064.012480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177064.012484:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177064.012485:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177064.012489:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177064.012489:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177064.012492:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177064.012575:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177064.012579:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177064.012583:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177064.652471:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177064.652479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177064.652480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177064.652486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177064.652489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177064.652493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177064.652496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177064.652498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177064.652502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177065.036463:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177065.036465:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177065.036471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177065.036473:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177065.036477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177065.036478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177065.036480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177065.036485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177065.036486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177065.036486:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177065.036493:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177065.036498:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177065.036501:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177065.036504:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177065.036508:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177065.036538:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177065.036541:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177065.036544:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177065.676465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177065.676472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177065.676474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177065.676478:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177065.676482:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177065.676485:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177065.676487:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177065.676489:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177065.676491:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177066.060468:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177066.060472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177066.060475:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177066.060478:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177066.060478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177066.060484:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177066.060486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177066.060489:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177066.060490:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177066.060495:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177066.060497:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177066.060500:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177066.060501:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177066.060506:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177066.060510:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177066.060558:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177066.060562:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177066.060565:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177066.700642:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177066.700649:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177066.700650:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177066.700654:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177066.700657:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177066.700660:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177066.700662:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177066.700664:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177066.700666:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177067.020455:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306413824, at 4306413888 00000100:00000010:0.0:1786177067.020470:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ba0. 00000100:00000040:1.0:1786177067.020476:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306413824, at 4306413888 00000100:00000010:1.0:1786177067.020485:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62c60. 00000100:00000001:1.0:1786177067.084467:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177067.084474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177067.084475:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177067.084477:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177067.084480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177067.084482:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177067.084487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177067.084489:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177067.084490:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177067.084493:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177067.084494:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177067.084497:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177067.084502:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177067.084502:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177067.084507:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177067.084572:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177067.084575:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177067.084578:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177067.724459:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177067.724469:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177067.724472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177067.724478:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177067.724482:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177067.724485:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177067.724486:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177067.724489:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177067.724491:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.108491:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177068.108496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177068.108499:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177068.108500:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.108502:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177068.108506:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177068.108508:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177068.108509:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.108514:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177068.108517:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177068.108518:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177068.108521:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.108522:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177068.108526:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.108530:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177068.108606:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177068.108609:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177068.108612:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177068.236489:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177068.236500:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177068.236503:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177068.236507:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177068.236512:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177068.236519:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11742 00000400:00000200:3.0:1786177068.236528:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177068.236534:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177068.236537:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177068.236541:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.236545:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177068.236547:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.236550:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177068.236554:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177068.236557:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177068.236560:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177068.236563:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177068.236566:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177068.236569:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177068.236572:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.236575:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177068.236590:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.236593:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177068.236596:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177068.236600:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177068.236603:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177068.236606:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177068.236609:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177068.236613:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177068.236615:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177068.236618:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177068.236633:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786177068.236637:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177068.236641:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177068.236643:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177068.236646:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177068.236650:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177068.236654:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177068.236661:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786177068.236668:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177068.236677:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177068.236688:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177068.236693:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177068.236698:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177068.236727:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177068.236731:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177068.236735:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177068.236737:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177068.236738:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177068.236741:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177068.236744:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11742 00000400:00000200:3.0:1786177068.236755:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177068.236760:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1786177068.236762:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177068.236765:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1786177068.236766:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177068.236768:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1786177068.236774:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177068.236777:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177068.236779:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177068.236786:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.236793:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177068.236794:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177068.236797:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177068.236799:0:40394:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1786177068.236802:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1786177068.236802:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177068.236805:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1786177068.236806:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1786177068.236810:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 02000000:00000001:1.0:1786177068.236811:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177068.236817:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00100000:1.0:1786177068.236820:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235571200:192.168.202.132@tcp:250:kworker.0 00000020:00000001:3.0:1786177068.236822:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177068.236831:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1786177068.236831:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.236836:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177068.236838:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1786177068.236839:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177068.236843:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177068.236844:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:1.0:1786177068.236850:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1786177068.236850:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177068.236855:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177068.236858:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177068.236859:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:3.0:1786177068.236864:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177068.236865:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177068.236871:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1786177068.236876:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:3.0:1786177068.236877:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177068.236881:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00000001:3.0:1786177068.236882:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177068.236888:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1786177068.236890:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000008:00000001:3.0:1786177068.236894:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177068.236898:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177068.236899:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235571200, portal 25 00000100:00000001:1.0:1786177068.236902:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000010:3.0:1786177068.236903:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000100:00000001:1.0:1786177068.236904:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280814464 : -110371428737152 : ffff9b9e26325f80) 00000020:00000040:3.0:1786177068.236911:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000100:00000040:1.0:1786177068.236914:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177068.236917:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177068.236920:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177068.236925:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177068.236928:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177068.236933:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.236935:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177068.236938:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 02000000:00000010:3.0:1786177068.236940:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000400:00000010:1.0:1786177068.236942:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177068.236946:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235571200, offset 0 00000100:00000001:3.0:1786177068.236947:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177068.236953:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00100000:3.0:1786177068.236957:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177068.236964:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177068.236973:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:3.0:1786177068.236975:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177068.236979:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1786177068.236981:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177068.236985:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:3.0:1786177068.236990:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177068.236992:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177068.236997:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177068.237003:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426529:426529:256:4294967295] 192.168.202.132@tcp LPNI seq info [426491:426491:8:4294967295] 00000400:00000200:1.0:1786177068.237013:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177068.237021:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177068.237027:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177068.237028:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00080000:3.0:1786177068.237037:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.0:1786177068.237040:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177068.237041:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177068.237045:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:1.0:1786177068.237047:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000001:3.0:1786177068.237049:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177068.237056:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11742 00000800:00000200:1.0:1786177068.237059:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:3.0:1786177068.237065:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:1.0:1786177068.237066:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000400:00000200:3.0:1786177068.237073:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177068.237080:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1786177068.237080:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177068.237088:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000100:00000040:3.0:1786177068.237090:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.237097:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:1.0:1786177068.237101:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177068.237102:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000010:1.0:1786177068.237105:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00000001:3.0:1786177068.237107:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177068.237112:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1786177068.237113:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:3.0:1786177068.237114:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177068.237118:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1786177068.237121:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000020:00000040:3.0:1786177068.237122:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:1.0:1786177068.237127:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000020:00000001:3.0:1786177068.237128:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000010:1.0:1786177068.237131:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000020:00000001:3.0:1786177068.237133:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000001:1.0:1786177068.237136:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1786177068.237140:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000200:1.0:1786177068.237143:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177068.237145:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177068.237151:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.237158:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177068.237160:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1786177068.237161:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00000040:1.0:1786177068.237166:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177068.237168:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177068.237176:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177068.237181:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177068.237186:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177068.237188:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177068.237190:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.237191:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1786177068.237196:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177068.237198:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1786177068.237199:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177068.237205:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000008:00000001:3.0:1786177068.237205:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177068.237210:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.237211:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306418944 00000100:00000010:3.0:1786177068.237218:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000100:00000010:1.0:1786177068.237219:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62c60. 00000020:00000040:3.0:1786177068.237223:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000001:1.0:1786177068.237227:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1786177068.237229:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177068.237232:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177068.237234:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177068.237237:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306418944 02000000:00000001:3.0:1786177068.237237:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177068.237241:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177068.237243:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62840. 00000100:00000001:1.0:1786177068.237248:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 02000000:00000010:3.0:1786177068.237250:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:1.0:1786177068.237251:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177068.237255:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177068.237258:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.237259:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177068.237264:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177068/real 1786177068] req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786177068.237269:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177068.237284:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177068.237289:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1786177068.237291:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177068.237294:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1786177068.237296:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1786177068.237297:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177068.237302:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177068.237305:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177068.237309:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177068.237315:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177068.237316:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177068.237322:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1786177068.237328:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177068.237331:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177068.237336:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177068.237337:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177068.237340:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177068.237340:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177068.237344:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177068.237348:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177068.237348:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177068.237378:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11742 00000400:00000200:3.0:1786177068.237384:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:1.0:1786177068.237387:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1786177068.237391:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177068.237398:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177068.237403:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177068.237405:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.237409:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177068.237410:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177068.237415:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.237421:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177068.237423:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1786177068.237427:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177068.237428:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.237431:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:3.0:1786177068.237431:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:1.0:1786177068.237434:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1786177068.237436:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177068.237438:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000040:3.0:1786177068.237440:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:1.0:1786177068.237441:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1786177068.237447:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00080000:1.0:1786177068.237448:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177068.237455:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000020:00000001:3.0:1786177068.237455:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177068.237462:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1786177068.237464:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177068.237469:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177068.237470:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177068.237477:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.237483:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00100000:1.0:1786177068.237486:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235571200:192.168.202.132@tcp:250:kworker.0 00000020:00000040:3.0:1786177068.237488:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:1.0:1786177068.237492:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177068.237495:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177068.237496:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1786177068.237502:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177068.237503:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325f80 x1872930235571200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177068.237508:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177068.237512:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1786177068.237513:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177068.237515:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177068.237519:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:3.0:1786177068.237520:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177068.237524:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1786177068.237525:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000020:00000001:1.0:1786177068.237526:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177068.237529:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000008:00100000:3.0:1786177068.237530:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000020:00000001:1.0:1786177068.237532:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177068.237535:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa000. 00000008:00000001:3.0:1786177068.237536:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177068.237540:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1786177068.237541:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177068.237544:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177068.237546:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325f80. 00000100:00000010:3.0:1786177068.237548:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000100:00000001:1.0:1786177068.237550:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177068.237553:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1786177068.237553:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177068.237557:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1786177068.237559:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177068.237562:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.237565:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177068.237566:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177068.237569:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177068.237570:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177068.237573:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1786177068.237575:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.237576:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177068.237592:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:3.0:1786177068.237596:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000040:1.0:1786177068.237597:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177068.237603:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.237610:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177068.237612:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177068.237617:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177068.237618:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177068.237622:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177068.237628:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177068.237636:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1786177068.237638:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571328:192.168.202.132@tcp:38:kworker.0 00000100:00000001:3.0:1786177068.237640:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.237647:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1786177068.237649:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177068.237652:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177068.237655:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177068.237660:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1786177068.237668:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177068.237671:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:1.0:1786177068.237672:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177068.237676:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177068.237678:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177068.237683:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1786177068.237684:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:3.0:1786177068.237688:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1786177068.237690:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00000001:3.0:1786177068.237693:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:1.0:1786177068.237699:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00080000:3.0:1786177068.237699:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11742 00000100:00000200:1.0:1786177068.237707:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235571328, portal 10 00000400:00000200:3.0:1786177068.237708:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177068.237713:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000400:00000200:3.0:1786177068.237716:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177068.237717:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280820736 : -110371428730880 : ffff9b9e26327800) 00000100:00080000:3.0:1786177068.237721:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177068.237727:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177068.237731:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177068.237732:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177068.237739:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.237745:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177068.237748:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1786177068.237751:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177068.237754:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000040:1.0:1786177068.237756:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000020:00000001:3.0:1786177068.237757:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:1.0:1786177068.237761:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000020:00000040:3.0:1786177068.237762:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000200:1.0:1786177068.237767:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235571328, offset 0 00000020:00000001:3.0:1786177068.237768:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177068.237775:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000200:1.0:1786177068.237780:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177068.237784:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.237787:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177068.237789:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177068.237792:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177068.237796:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177068.237798:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000040:3.0:1786177068.237800:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000400:00000200:1.0:1786177068.237805:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1786177068.237807:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:1.0:1786177068.237813:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:3.0:1786177068.237813:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177068.237819:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1786177068.237820:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1786177068.237825:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1786177068.237826:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 02000000:00000001:3.0:1786177068.237831:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177068.237834:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426530:426530:256:4294967295] 192.168.202.132@tcp LPNI seq info [426492:426492:8:4294967295] 00000100:00000001:3.0:1786177068.237836:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177068.237840:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000200:1.0:1786177068.237845:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000008:00000001:3.0:1786177068.237847:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177068.237852:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177068.237860:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000010:3.0:1786177068.237860:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786177068.237866:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000800:00000200:1.0:1786177068.237872:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 02000000:00000001:3.0:1786177068.237873:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177068.237877:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177068.237879:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000001:3.0:1786177068.237880:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177068.237884:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177068.237887:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177068.237889:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000010:3.0:1786177068.237893:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000800:00000200:1.0:1786177068.237895:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000100:00000001:3.0:1786177068.237900:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786177068.237905:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177068.237910:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000100:00100000:3.0:1786177068.237911:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1786177068.237920:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177068.237925:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00000001:3.0:1786177068.237927:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177068.237930:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1786177068.237931:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177068.237940:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1786177068.237941:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177068.237944:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177068.237948:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177068.237954:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786177068.237960:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177068.237961:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177068.237966:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11752) 00000100:00000001:1.0:1786177068.237975:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177068.237979:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.237987:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.237989:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177068.237993:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177068.237997:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177068.238001:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177068.238003:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306415108 00000100:00000001:1.0:1786177068.238006:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306418944 00000100:00000001:1.0:1786177068.238009:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306415108 00000100:00000001:1.0:1786177068.238011:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306418944 00000100:00000001:1.0:1786177068.238013:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177068.238015:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.238017:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177068.238021:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177068/real 1786177068] req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177068.238028:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177068.238029:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177068.238031:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177068.238034:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177068.238035:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177068.238037:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177068.238040:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177068.238049:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238055:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177068.238057:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238058:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177068.238060:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177068.238063:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238069:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177068.238073:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238080:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177068.238083:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238084:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177068.238086:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238088:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177068.238090:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177068.238093:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177068.238096:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177068.238098:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177068.238102:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177068.238110:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571328:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177068.238113:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177068.238116:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238123:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177068.238124:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177068.238126:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177068.238128:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177068.238130:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177068.238133:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571456:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177068.238136:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177068.238138:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177068.238140:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177068.238141:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177068.238145:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177068.238147:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177068.238150:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177068.238152:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177068.238155:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177068.238158:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235571456, portal 10 00000100:00000001:1.0:1786177068.238159:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177068.238162:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818048 : -110371428733568 : ffff9b9e26326d80) 00000100:00000040:1.0:1786177068.238167:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238173:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177068.238175:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177068.238177:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177068.238180:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235571456, offset 0 00000400:00000200:1.0:1786177068.238183:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177068.238187:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177068.238191:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177068.238194:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177068.238197:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177068.238199:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177068.238201:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177068.238204:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426531:426531:256:4294967295] 192.168.202.132@tcp LPNI seq info [426493:426493:8:4294967295] 00000400:00000200:1.0:1786177068.238209:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177068.238216:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177068.238221:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177068.238224:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177068.238228:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177068.238232:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786177068.238236:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177068.238239:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1786177068.238243:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177068.238245:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177068.238248:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177068.238252:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177068.238254:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177068.238256:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177068.238257:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177068.238260:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238266:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177068.238268:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238272:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.238273:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177068.238276:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177068.238278:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177068.238280:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177068.238282:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306415108 00000100:00000001:1.0:1786177068.238284:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306418944 00000100:00000001:1.0:1786177068.238285:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306415108 00000100:00000001:1.0:1786177068.238287:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306418944 00000100:00000001:1.0:1786177068.238288:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177068.238289:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.238290:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177068.238293:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177068/real 1786177068] req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177068.238298:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177068.238299:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177068.238300:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177068.238301:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177068.238303:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177068.238304:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177068.238306:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177068.238311:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238316:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177068.238317:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238318:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177068.238319:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177068.238322:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238326:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177068.238328:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238332:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177068.238334:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238335:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177068.238335:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238336:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177068.238338:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177068.238340:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177068.238342:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177068.238344:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177068.238346:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177068.238351:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571456:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177068.238353:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238354:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177068.238372:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327800 x1872930235571328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238377:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177068.238379:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177068.238380:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177068.238382:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177068.238383:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177068.238384:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177068.238385:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177068.238386:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa800. 02000000:00000001:1.0:1786177068.238388:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177068.238389:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177068.238390:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327800. 00000100:00000001:1.0:1786177068.238392:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177068.238393:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238394:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177068.238396:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326d80 x1872930235571456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238401:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177068.238401:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177068.238402:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177068.238404:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177068.238405:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177068.238405:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177068.238406:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177068.238407:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9000. 02000000:00000001:1.0:1786177068.238408:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177068.238409:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177068.238410:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326d80. 00000100:00000001:1.0:1786177068.238411:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177068.238412:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238413:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238416:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177068.238417:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.238418:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177068.238419:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177068.238420:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177068.238422:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238426:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177068.238428:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177068.238429:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177068.238430:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177068.238432:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177068.238433:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571584:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177068.238435:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177068.238437:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177068.238438:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177068.238439:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177068.238441:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177068.238442:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177068.238445:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177068.238446:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177068.238448:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177068.238450:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235571584, portal 4 00000100:00000001:1.0:1786177068.238452:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177068.238453:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280810880 : -110371428740736 : ffff9b9e26325180) 00000100:00000040:1.0:1786177068.238457:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238461:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177068.238463:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177068.238464:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177068.238466:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235571584, offset 0 00000400:00000200:1.0:1786177068.238468:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177068.238471:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177068.238473:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177068.238475:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177068.238477:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177068.238478:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177068.238480:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177068.238482:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426532:426532:256:4294967295] 192.168.202.132@tcp LPNI seq info [426494:426494:8:4294967295] 00000400:00000200:1.0:1786177068.238485:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177068.238489:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177068.238493:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177068.238495:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177068.238497:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177068.238500:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786177068.238504:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177068.238507:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1786177068.238511:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177068.238512:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177068.238515:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177068.238518:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177068.238521:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177068.238523:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177068.238525:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177068.238529:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238537:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177068.238541:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238548:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.238550:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177068.238554:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177068.238557:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177068.238560:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177068.238562:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306415109 00000100:00000001:1.0:1786177068.238564:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306418944 00000100:00000001:1.0:1786177068.238566:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306415109 00000100:00000001:1.0:1786177068.238567:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306418944 00000100:00000001:1.0:1786177068.238568:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177068.238570:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.238571:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177068.238574:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177068/real 1786177068] req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177068.238596:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177068.238597:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177068.238598:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177068.238600:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177068.238601:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177068.238602:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177068.238605:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177068.238611:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238616:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177068.238617:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238619:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177068.238620:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177068.238624:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238631:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177068.238635:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238642:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177068.238645:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238646:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177068.238648:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238650:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177068.238652:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177068.238655:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177068.238658:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177068.238660:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177068.238665:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177068.238671:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571584:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177068.238674:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177068.238676:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238681:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177068.238682:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177068.238685:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177068.238687:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177068.238689:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177068.238692:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571712:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177068.238695:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177068.238698:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177068.238699:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177068.238701:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177068.238705:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177068.238707:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177068.238711:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177068.238715:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177068.238719:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177068.238722:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235571712, portal 4 00000100:00000001:1.0:1786177068.238724:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177068.238725:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280813568 : -110371428738048 : ffff9b9e26325c00) 00000100:00000040:1.0:1786177068.238732:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238738:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177068.238740:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177068.238743:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177068.238746:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235571712, offset 0 00000400:00000200:1.0:1786177068.238751:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177068.238754:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177068.238756:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177068.238758:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177068.238760:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177068.238762:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177068.238763:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177068.238765:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426533:426533:256:4294967295] 192.168.202.132@tcp LPNI seq info [426495:426495:8:4294967295] 00000400:00000200:1.0:1786177068.238768:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177068.238773:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177068.238776:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177068.238778:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177068.238781:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177068.238792:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786177068.238796:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177068.238801:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1786177068.238803:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177068.238805:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177068.238807:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177068.238809:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177068.238812:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177068.238813:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177068.238814:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177068.238817:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238822:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177068.238824:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177093 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238828:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.238829:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177068.238831:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177068.238833:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177068.238836:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177068.238838:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306415111 00000100:00000001:1.0:1786177068.238840:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306418944 00000100:00000001:1.0:1786177068.238842:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306415111 00000100:00000001:1.0:1786177068.238843:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306418944 00000100:00000001:1.0:1786177068.238844:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177068.238845:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.238847:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177068.238849:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177068/real 1786177068] req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177068.238854:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177068.238855:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177068.238856:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177068.238858:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177068.238859:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177068.238860:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177068.238862:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177068.238868:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238872:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177068.238873:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238874:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177068.238875:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177068.238877:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238881:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177068.238883:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238887:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177068.238889:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238890:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177068.238891:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238892:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177068.238893:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177068.238895:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177068.238897:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177068.238898:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177068.238901:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177068.238905:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571712:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177068.238907:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238909:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177068.238910:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325180 x1872930235571584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238915:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177068.238916:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177068.238917:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177068.238918:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177068.238919:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177068.238921:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177068.238922:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177068.238923:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9400. 02000000:00000001:1.0:1786177068.238925:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177068.238926:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177068.238927:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325180. 00000100:00000001:1.0:1786177068.238928:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177068.238929:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238930:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177068.238932:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325c00 x1872930235571712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177093 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177068.238940:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177068.238941:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177068.238942:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177068.238944:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177068.238944:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177068.238945:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177068.238946:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177068.238947:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:1.0:1786177068.238948:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177068.238949:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177068.238949:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325c00. 00000100:00000001:1.0:1786177068.238951:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177068.238951:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238952:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177068.238955:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177068.238956:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177068.238957:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177068.238958:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177068.748469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177068.748477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177068.748479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177068.748486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177068.748491:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177068.748495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177068.748498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177068.748501:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177068.748505:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177069.132475:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177069.132483:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177069.132486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177069.132495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177069.132497:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177069.132503:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177069.132504:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177069.132507:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177069.132511:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177069.132514:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177069.132516:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177069.132517:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177069.132521:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177069.132526:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177069.132530:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177069.132596:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177069.132601:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177069.132604:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177069.772461:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177069.772469:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177069.772472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177069.772479:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177069.772483:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177069.772488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177069.772492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177069.772495:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177069.772499:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177070.156465:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177070.156468:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177070.156471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177070.156476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177070.156479:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177070.156480:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177070.156483:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177070.156483:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177070.156485:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177070.156488:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177070.156490:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177070.156491:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177070.156492:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177070.156498:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177070.156499:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177070.156499:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177070.156506:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177070.156509:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177070.156512:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177070.156515:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177070.156519:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177070.156612:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177070.156615:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177070.156618:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177070.796473:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177070.796482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177070.796485:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177070.796491:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177070.796495:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177070.796500:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177070.796503:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177070.796506:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177070.796509:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177071.180456:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177071.180460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177071.180463:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177071.180464:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177071.180466:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177071.180471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177071.180472:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177071.180474:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177071.180477:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177071.180479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177071.180482:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177071.180482:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177071.180488:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177071.180491:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177071.180494:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177071.180577:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177071.180580:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177071.180583:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177071.820467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177071.820476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177071.820479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177071.820485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177071.820490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177071.820493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177071.820496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177071.820499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177071.820504:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177072.140464:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306418944, at 4306419008 00000100:00000040:1.0:1786177072.140464:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306418944, at 4306419008 00000100:00000010:0.0:1786177072.140478:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62c60. 00000100:00000010:1.0:1786177072.140484:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62840. 00000100:00000001:1.0:1786177072.204457:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177072.204460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177072.204467:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177072.204469:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177072.204470:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177072.204472:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177072.204474:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177072.204479:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:1.0:1786177072.204480:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177072.204481:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177072.204487:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177072.204487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177072.204492:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177072.204492:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177072.204497:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177072.204498:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177072.204501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177072.204502:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177072.204507:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177072.204508:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177072.204511:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177072.204512:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177072.204514:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177072.204515:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177072.204516:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177072.204517:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177072.204519:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177072.204586:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177072.204591:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177072.204595:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177072.844469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177072.844478:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177072.844480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177072.844485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177072.844489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177072.844492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177072.844494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177072.844496:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177072.844498:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.228496:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177073.228500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177073.228503:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177073.228504:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.228506:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177073.228512:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177073.228513:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177073.228516:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.228518:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177073.228521:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177073.228522:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177073.228524:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.228526:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177073.228552:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.228556:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177073.228615:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177073.228619:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177073.228622:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177073.356460:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177073.356468:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177073.356470:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177073.356473:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177073.356477:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177073.356483:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11747 00000400:00000200:3.0:1786177073.356490:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177073.356495:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177073.356497:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177073.356500:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177073.356503:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177073.356505:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177073.356506:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177073.356510:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177073.356512:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177073.356513:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177073.356516:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177073.356518:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177073.356519:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177073.356521:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177073.356523:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177073.356524:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177073.356525:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177073.356551:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177073.356554:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177073.356556:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177073.356558:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177073.356561:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177073.356563:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177073.356564:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177073.356567:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177073.356575:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786177073.356577:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177073.356580:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177073.356582:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177073.356584:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177073.356587:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177073.356590:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177073.356596:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786177073.356601:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177073.356620:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177073.356628:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177073.356632:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177073.356635:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177073.356652:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177073.356654:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177073.356657:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177073.356661:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177073.356662:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177073.356664:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177073.356666:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11747 00000400:00000200:3.0:1786177073.356669:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177073.356671:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177073.356673:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177073.356676:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177073.356677:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177073.356679:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177073.356681:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177073.356683:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177073.356683:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177073.356684:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177073.356686:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177073.356687:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177073.356689:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177073.356691:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177073.356693:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177073.356694:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177073.356696:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177073.356698:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177073.356701:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177073.356703:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177073.356704:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177073.356708:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1786177073.356709:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177073.356714:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.356715:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177073.356716:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177073.356721:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1786177073.356722:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177073.356727:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177073.356732:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177073.356736:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177073.356742:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786177073.356746:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177073.356751:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177073.356753:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.356754:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177073.356755:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177073.356759:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177073.356759:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177073.356760:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177073.356768:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:3.0:1786177073.356771:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 02000000:00000001:1.0:1786177073.356773:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1786177073.356775:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177073.356778:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177073.356783:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1786177073.356787:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235571840:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177073.356796:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1786177073.356800:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1786177073.356801:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1786177073.356802:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177073.356805:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177073.356808:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1786177073.356812:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177073.356813:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177073.356816:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177073.356825:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177073.356830:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00000001:3.0:1786177073.356833:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177073.356836:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00080000:3.0:1786177073.356838:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1786177073.356844:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235571840, portal 25 00000100:00080000:3.0:1786177073.356845:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177073.356848:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1786177073.356849:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177073.356850:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280815360 : -110371428736256 : ffff9b9e26326300) 00000100:00080000:3.0:1786177073.356854:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1786177073.356859:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177073.356861:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177073.356868:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11747 00000100:00000001:1.0:1786177073.356869:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177073.356875:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000200:3.0:1786177073.356879:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177073.356880:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177073.356884:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235571840, offset 0 00000400:00000200:3.0:1786177073.356888:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177073.356893:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177073.356893:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177073.356900:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177073.356902:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177073.356906:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177073.356910:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000040:3.0:1786177073.356913:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177073.356917:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1786177073.356918:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177073.356922:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1786177073.356924:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000040:3.0:1786177073.356925:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177073.356928:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1786177073.356930:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1786177073.356931:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:1.0:1786177073.356936:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000001:3.0:1786177073.356936:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177073.356939:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177073.356943:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426534:426534:256:4294967295] 192.168.202.132@tcp LPNI seq info [426496:426496:8:4294967295] 00000100:00000040:3.0:1786177073.356947:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177073.356951:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177073.356960:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:3.0:1786177073.356962:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1786177073.356966:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177073.356968:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000010:1.0:1786177073.356971:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000001:3.0:1786177073.356972:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177073.356978:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000200:1.0:1786177073.356979:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177073.356985:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000100:00080000:3.0:1786177073.356986:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:1.1:1786177073.356993:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177073.356994:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786177073.356999:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 02000000:00000001:3.0:1786177073.357000:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:1.0:1786177073.357006:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 02000000:00000001:3.0:1786177073.357007:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1786177073.357008:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 02000000:00000001:3.0:1786177073.357014:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177073.357016:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1786177073.357018:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1786177073.357022:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000008:00100000:3.0:1786177073.357024:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177073.357027:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000008:00000001:3.0:1786177073.357029:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177073.357030:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:3.0:1786177073.357035:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357036:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:3.0:1786177073.357040:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000100:00000200:1.0:1786177073.357041:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177073.357045:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177073.357051:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177073.357055:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357057:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1786177073.357057:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177073.357060:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1786177073.357062:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177073.357063:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177073.357070:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:1.0:1786177073.357072:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177073.357074:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357075:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177073.357081:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177073.357082:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177073.357088:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177073.357093:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306424064 00000100:00000001:3.0:1786177073.357096:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000010:1.0:1786177073.357100:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62840. 00000100:00000001:3.0:1786177073.357100:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357109:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000040:3.0:1786177073.357109:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.357117:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306424064 00000100:00000001:3.0:1786177073.357119:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177073.357122:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62540. 00000100:00080000:3.0:1786177073.357123:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177073.357126:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1786177073.357128:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177073.357131:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177073.357132:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1786177073.357134:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177073.357135:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177073.357138:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177073.357139:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1786177073.357142:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11747 00000100:00100000:1.0:1786177073.357147:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177073/real 1786177073] req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1786177073.357152:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177073.357159:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177073.357164:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177073.357169:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177073.357176:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1786177073.357177:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1786177073.357179:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000001:1.0:1786177073.357180:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177073.357183:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000100:00000001:3.0:1786177073.357184:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177073.357186:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000020:00000001:3.0:1786177073.357188:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:1.0:1786177073.357189:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000020:00000040:3.0:1786177073.357190:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:1.0:1786177073.357193:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1786177073.357193:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177073.357197:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000200:1.0:1786177073.357200:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177073.357200:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177073.357206:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177073.357215:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000200:1.0:1786177073.357219:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177073.357220:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177073.357228:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177073.357229:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177073.357232:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177073.357233:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177073.357235:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177073.357237:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1786177073.357240:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00100000:1.0:1786177073.357242:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177073.357249:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177073.357251:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177073.357256:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177073.357257:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177073.357262:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177073.357267:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1786177073.357267:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177073.357272:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357273:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.357278:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1786177073.357278:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177073.357282:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1786177073.357283:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177073.357284:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00000001:3.0:1786177073.357287:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357288:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177073.357289:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177073.357292:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000010:3.0:1786177073.357295:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000100:00080000:1.0:1786177073.357297:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000020:00000040:3.0:1786177073.357299:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177073.357300:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1786177073.357303:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1786177073.357306:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177073.357306:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177073.357309:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177073.357312:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177073.357316:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177073.357320:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235571840:192.168.202.132@tcp:250:kworker.0 02000000:00000010:3.0:1786177073.357324:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:1.0:1786177073.357325:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.357328:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177073.357332:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326300 x1872930235571840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177073.357332:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357341:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177073.357343:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:3.0:1786177073.357345:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1786177073.357346:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177073.357350:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177073.357353:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177073.357355:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000001:3.0:1786177073.357386:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177073.357389:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177073.357392:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177073.357396:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 00000100:00000040:3.0:1786177073.357396:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177073.357401:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177073.357403:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177073.357406:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326300. 00000100:00000001:3.0:1786177073.357414:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357420:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00080000:3.0:1786177073.357421:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177073.357425:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177073.357427:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177073.357430:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177073.357431:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177073.357435:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177073.357439:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177073.357442:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1786177073.357442:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11747 00000400:00000200:3.0:1786177073.357452:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177073.357454:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1786177073.357457:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177073.357460:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1786177073.357462:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177073.357464:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1786177073.357468:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177073.357469:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177073.357474:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177073.357476:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177073.357480:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177073.357486:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177073.357487:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177073.357490:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177073.357492:0:191280:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1786177073.357493:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177073.357497:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1786177073.357497:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 02000000:00000001:1.0:1786177073.357502:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177073.357502:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177073.357507:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00100000:1.0:1786177073.357508:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571968:192.168.202.132@tcp:38:kworker.0 00000020:00000001:3.0:1786177073.357512:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177073.357514:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1786177073.357517:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 02000000:00000001:1.0:1786177073.357518:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177073.357520:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177073.357524:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1786177073.357525:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000010:1.0:1786177073.357543:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac800. 00000100:00000001:3.0:1786177073.357545:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177073.357546:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177073.357551:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177073.357553:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177073.357557:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:1.0:1786177073.357558:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177073.357565:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000020:00000040:3.0:1786177073.357565:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177073.357571:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177073.357577:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177073.357578:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235571968, portal 10 02000000:00000001:3.0:1786177073.357579:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177073.357583:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1786177073.357586:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177073.357587:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280821632 : -110371428729984 : ffff9b9e26327b80) 02000000:00000001:3.0:1786177073.357591:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177073.357596:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:1.0:1786177073.357600:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1786177073.357601:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177073.357608:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177073.357613:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357620:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:3.0:1786177073.357621:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000100:00000040:1.0:1786177073.357625:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177073.357628:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000400:00000010:1.0:1786177073.357630:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 02000000:00000001:3.0:1786177073.357632:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1786177073.357634:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235571968, offset 0 02000000:00000001:3.0:1786177073.357636:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177073.357640:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177073.357642:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177073.357645:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1786177073.357647:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:3.0:1786177073.357649:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177073.357652:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177073.357656:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:3.0:1786177073.357657:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000400:00000200:1.0:1786177073.357660:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177073.357664:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1786177073.357664:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177073.357670:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177073.357677:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426535:426535:256:4294967295] 192.168.202.132@tcp LPNI seq info [426497:426497:8:4294967295] 00000100:00100000:3.0:1786177073.357677:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177073.357689:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177073.357697:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177073.357702:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177073.357705:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000040:3.0:1786177073.357710:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1786177073.357717:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177073.357724:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000001:3.0:1786177073.357730:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177073.357731:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786177073.357736:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11757) 00000800:00000200:1.0:1786177073.357737:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786177073.357746:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177073.357749:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1786177073.357754:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177073.357756:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177073.357760:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177073.357765:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177073.357769:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177073.357771:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177073.357774:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177073.357778:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.357784:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177073.357786:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.357791:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357792:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177073.357794:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177073.357797:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177073.357799:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177073.357801:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306420228 00000100:00000001:1.0:1786177073.357803:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306424064 00000100:00000001:1.0:1786177073.357804:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306420228 00000100:00000001:1.0:1786177073.357806:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306424064 00000100:00000001:1.0:1786177073.357807:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177073.357809:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357810:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177073.357812:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177073/real 1786177073] req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177073.357818:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177073.357819:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177073.357820:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177073.357822:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177073.357823:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177073.357825:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177073.357827:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177073.357832:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.357835:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177073.357836:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.357838:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177073.357839:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177073.357841:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.357846:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177073.357848:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.357852:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177073.357854:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.357855:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177073.357855:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.357857:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177073.357858:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177073.357860:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177073.357862:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177073.357863:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177073.357866:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177073.357871:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235571968:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177073.357873:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177073.357876:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.357880:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177073.357881:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177073.357882:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177073.357884:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177073.357885:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177073.357887:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572096:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177073.357889:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177073.357890:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177073.357891:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177073.357892:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177073.357895:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177073.357896:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177073.357898:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177073.357900:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177073.357902:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177073.357904:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235572096, portal 10 00000100:00000001:1.0:1786177073.357905:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177073.357906:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280812672 : -110371428738944 : ffff9b9e26325880) 00000100:00000040:1.0:1786177073.357910:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.357914:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177073.357916:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177073.357917:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177073.357919:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235572096, offset 0 00000400:00000200:1.0:1786177073.357921:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177073.357924:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177073.357926:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177073.357928:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177073.357931:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177073.357932:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177073.357934:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177073.357936:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426536:426536:256:4294967295] 192.168.202.132@tcp LPNI seq info [426498:426498:8:4294967295] 00000400:00000200:1.0:1786177073.357939:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177073.357943:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177073.357945:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177073.357947:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177073.357950:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177073.357952:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786177073.357955:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177073.357956:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1786177073.357959:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177073.357960:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177073.357962:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177073.357964:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177073.357967:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177073.357968:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177073.357969:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177073.357972:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.357976:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177073.357978:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.357983:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357983:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177073.357985:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177073.357987:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177073.357989:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177073.357990:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306420228 00000100:00000001:1.0:1786177073.357991:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306424064 00000100:00000001:1.0:1786177073.357993:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306420228 00000100:00000001:1.0:1786177073.357994:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306424064 00000100:00000001:1.0:1786177073.357996:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177073.357997:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.357998:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177073.358001:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177073/real 1786177073] req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177073.358005:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177073.358006:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177073.358007:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177073.358008:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177073.358009:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177073.358010:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177073.358012:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177073.358018:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358022:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177073.358023:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358024:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177073.358025:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177073.358027:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358031:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177073.358033:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358037:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177073.358038:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358039:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177073.358040:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358041:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177073.358043:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177073.358045:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177073.358047:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177073.358048:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177073.358051:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177073.358055:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572096:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177073.358057:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177073.358059:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358063:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177073.358064:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177073.358065:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177073.358066:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177073.358067:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177073.358069:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572224:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177073.358071:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177073.358072:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177073.358073:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177073.358074:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177073.358076:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177073.358078:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177073.358080:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177073.358081:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177073.358083:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177073.358085:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235572224, portal 4 00000100:00000001:1.0:1786177073.358087:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177073.358087:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280811776 : -110371428739840 : ffff9b9e26325500) 00000100:00000040:1.0:1786177073.358091:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358096:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177073.358098:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177073.358099:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177073.358101:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235572224, offset 0 00000400:00000200:1.0:1786177073.358103:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177073.358105:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177073.358107:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177073.358108:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177073.358110:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177073.358111:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177073.358113:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177073.358115:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426537:426537:256:4294967295] 192.168.202.132@tcp LPNI seq info [426499:426499:8:4294967295] 00000400:00000200:1.0:1786177073.358118:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177073.358122:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177073.358124:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177073.358126:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177073.358128:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177073.358130:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786177073.358132:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177073.358134:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1786177073.358136:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177073.358138:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177073.358139:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177073.358142:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177073.358143:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177073.358144:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177073.358146:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177073.358148:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358153:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177073.358155:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358163:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.358164:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177073.358166:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177073.358168:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177073.358169:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177073.358171:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306420228 00000100:00000001:1.0:1786177073.358172:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306424064 00000100:00000001:1.0:1786177073.358174:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306420228 00000100:00000001:1.0:1786177073.358175:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306424064 00000100:00000001:1.0:1786177073.358176:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177073.358177:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.358179:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177073.358181:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177073/real 1786177073] req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177073.358185:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177073.358186:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177073.358187:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177073.358188:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177073.358189:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177073.358190:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177073.358192:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177073.358197:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358201:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177073.358202:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358203:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177073.358204:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177073.358206:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358211:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177073.358213:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358217:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177073.358219:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358220:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177073.358221:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358222:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177073.358223:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177073.358224:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177073.358226:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177073.358227:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177073.358229:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177073.358234:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572224:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177073.358236:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358237:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177073.358239:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327b80 x1872930235571968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358243:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177073.358244:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177073.358245:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177073.358247:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177073.358247:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177073.358248:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177073.358250:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177073.358251:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:1.0:1786177073.358253:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177073.358254:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177073.358255:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327b80. 00000100:00000001:1.0:1786177073.358257:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177073.358258:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358259:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177073.358261:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325880 x1872930235572096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358265:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177073.358266:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177073.358267:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177073.358268:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177073.358268:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177073.358269:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177073.358270:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177073.358271:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:1.0:1786177073.358272:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177073.358273:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177073.358274:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325880. 00000100:00000001:1.0:1786177073.358275:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177073.358276:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358277:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177073.358279:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325500 x1872930235572224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358282:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177073.358283:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177073.358284:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177073.358285:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177073.358286:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177073.358286:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177073.358287:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177073.358288:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:1.0:1786177073.358290:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177073.358290:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177073.358291:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325500. 00000100:00000001:1.0:1786177073.358292:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177073.358293:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358295:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358297:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177073.358298:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.358300:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177073.358301:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177073.358302:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177073.358304:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358308:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177073.358309:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177073.358311:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177073.358312:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177073.358313:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177073.358315:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572352:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177073.358317:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177073.358318:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177073.358319:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177073.358320:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177073.358322:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177073.358323:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177073.358325:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177073.358327:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177073.358329:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177073.358330:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235572352, portal 4 00000100:00000001:1.0:1786177073.358332:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177073.358333:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280806400 : -110371428745216 : ffff9b9e26324000) 00000100:00000040:1.0:1786177073.358336:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358341:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177073.358342:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177073.358344:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177073.358345:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235572352, offset 0 00000400:00000200:1.0:1786177073.358347:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177073.358350:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177073.358352:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177073.358354:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177073.358355:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177073.358374:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177073.358375:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177073.358377:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426538:426538:256:4294967295] 192.168.202.132@tcp LPNI seq info [426500:426500:8:4294967295] 00000400:00000200:1.0:1786177073.358380:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177073.358384:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177073.358387:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177073.358390:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177073.358392:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177073.358394:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1786177073.358397:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177073.358399:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1786177073.358401:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177073.358403:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177073.358405:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177073.358407:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177073.358409:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177073.358411:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177073.358413:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177073.358418:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358426:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177073.358428:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177098 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358435:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.358437:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177073.358440:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177073.358443:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177073.358446:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177073.358449:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306420231 00000100:00000001:1.0:1786177073.358452:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306424064 00000100:00000001:1.0:1786177073.358454:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306420231 00000100:00000001:1.0:1786177073.358456:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306424064 00000100:00000001:1.0:1786177073.358458:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177073.358460:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.358462:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177073.358467:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177073/real 1786177073] req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177073.358472:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177073.358474:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177073.358475:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177073.358476:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177073.358478:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177073.358479:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177073.358481:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177073.358487:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358490:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177073.358491:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358492:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177073.358493:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177073.358495:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358500:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177073.358502:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358507:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177073.358509:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358510:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177073.358511:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358512:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177073.358513:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177073.358515:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177073.358517:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177073.358518:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177073.358520:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177073.358525:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572352:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177073.358543:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358545:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177073.358548:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324000 x1872930235572352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177098 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177073.358553:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177073.358554:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177073.358555:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177073.358557:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177073.358558:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177073.358559:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177073.358560:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177073.358561:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:1.0:1786177073.358563:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177073.358564:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177073.358565:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324000. 00000100:00000001:1.0:1786177073.358566:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177073.358567:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358569:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177073.358571:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177073.358572:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177073.358574:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177073.358576:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177073.868485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177073.868496:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177073.868500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177073.868506:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177073.868510:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177073.868514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177073.868517:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177073.868519:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177073.868555:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177074.252476:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177074.252483:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177074.252485:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177074.252490:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177074.252493:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177074.252497:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177074.252500:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177074.252502:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177074.252504:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177074.252514:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177074.252536:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177074.252545:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177074.252546:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177074.252551:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177074.252552:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177074.252660:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177074.252664:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177074.252668:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177074.892473:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177074.892481:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177074.892484:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177074.892490:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177074.892494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177074.892498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177074.892501:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177074.892504:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177074.892507:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177075.276500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177075.276506:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177075.276507:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177075.276525:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177075.276527:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177075.276531:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177075.276532:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177075.276536:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177075.276536:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177075.276543:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177075.276546:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177075.276549:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177075.276551:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177075.276553:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177075.276555:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177075.276612:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177075.276615:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177075.276618:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177075.916438:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177075.916447:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177075.916450:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177075.916456:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177075.916461:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177075.916472:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177075.916476:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177075.916479:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177075.916482:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177076.300466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177076.300468:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177076.300473:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177076.300476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177076.300478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177076.300480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177076.300482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177076.300483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177076.300539:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177076.300543:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177076.300547:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177076.300550:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177076.300553:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177076.300555:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177076.300558:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177076.300564:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177076.300568:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177076.300571:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177076.940508:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177076.940516:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177076.940520:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177076.940525:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177076.940528:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177076.940532:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177076.940535:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177076.940537:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177076.940539:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177077.260485:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306424064, at 4306424128 00000100:00000040:1.0:1786177077.260485:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306424064, at 4306424128 00000100:00000010:1.0:1786177077.260511:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62540. 00000100:00000010:0.0:1786177077.260514:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62840. 00010000:00000001:0.0:1786177077.324461:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177077.324467:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177077.324469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177077.324472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177077.324475:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177077.324476:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177077.324480:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177077.324481:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177077.324485:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177077.324487:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177077.324521:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177077.324525:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177077.324529:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177077.324532:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177077.324535:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177077.324547:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177077.324550:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177077.324554:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177077.964496:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177077.964504:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177077.964505:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177077.964510:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177077.964512:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177077.964515:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177077.964518:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177077.964520:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177077.964522:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177078.348465:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177078.348467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177078.348471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177078.348474:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177078.348476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177078.348478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177078.348499:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177078.348502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177078.348504:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177078.348508:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177078.348509:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177078.348510:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177078.348517:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177078.348521:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177078.348524:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177078.348579:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177078.348584:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177078.348587:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177078.476456:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177078.476464:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177078.476466:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177078.476469:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177078.476473:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177078.476492:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11752 00000400:00000200:1.0:1786177078.476500:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177078.476506:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177078.476508:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177078.476511:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476513:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177078.476515:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476517:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177078.476520:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177078.476521:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177078.476524:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177078.476527:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177078.476528:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177078.476530:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177078.476532:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476534:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177078.476535:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476536:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177078.476538:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177078.476540:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476542:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177078.476544:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177078.476546:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177078.476548:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.476550:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177078.476552:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177078.476559:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8700. 00000020:00000040:1.0:1786177078.476561:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177078.476564:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177078.476566:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.476568:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177078.476570:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177078.476574:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177078.476579:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac000. 00000100:00000001:1.0:1786177078.476584:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177078.476592:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177078.476599:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177078.476602:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177078.476606:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007a756761] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177078.476624:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177078.476626:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177078.476629:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177078.476630:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177078.476631:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177078.476634:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177078.476635:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11752 00000400:00000200:1.0:1786177078.476637:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177078.476639:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177078.476641:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177078.476643:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476645:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177078.476648:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476649:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177078.476651:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177078.476651:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177078.476652:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177078.476654:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177078.476655:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177078.476657:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177078.476660:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476661:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177078.476663:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476665:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177078.476665:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1786177078.476670:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:2.0:1786177078.476671:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1786177078.476673:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177078.476675:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177078.476676:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177078.476679:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177078.476682:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1786177078.476684:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177078.476685:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.476689:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177078.476694:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1786177078.476696:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1786177078.476699:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.476700:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.476703:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177078.476705:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:1.0:1786177078.476707:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8e00. 02000000:00000001:2.0:1786177078.476708:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1786177078.476710:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177078.476714:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177078.476714:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177078.476717:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177078.476721:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235572480:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177078.476722:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177078.476725:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177078.476728:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.476729:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:1.0:1786177078.476732:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac800. 02000000:00000001:2.0:1786177078.476733:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1786177078.476736:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177078.476737:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177078.476742:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1786177078.476746:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1786177078.476748:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177078.476754:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.476755:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177078.476759:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177078.476762:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ba93daf1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177078.476766:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177078.476774:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00000001:1.0:1786177078.476775:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177078.476778:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177078.476782:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177078.476783:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177078.476784:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1786177078.476785:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000001:1.0:1786177078.476786:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177078.476791:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11752 00000400:00000200:1.0:1786177078.476794:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000200:2.0:1786177078.476795:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235572480, portal 25 00000400:00000200:1.0:1786177078.476796:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177078.476801:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177078.476802:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1786177078.476804:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177078.476806:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487983872 : -110371221567744 : ffff9b9e328b8700) 00000100:00000001:1.0:1786177078.476807:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177078.476810:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476813:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177078.476815:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177078.476817:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000100:00000040:2.0:1786177078.476818:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177078.476819:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177078.476822:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177078.476826:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177078.476831:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000001:2.0:1786177078.476833:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177078.476836:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:2.0:1786177078.476840:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476841:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1786177078.476846:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000040:1.0:1786177078.476847:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476852:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177078.476854:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00000200:2.0:1786177078.476854:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235572480, offset 0 00000100:00080000:1.0:1786177078.476860:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476862:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177078.476865:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 02000000:00000001:1.0:1786177078.476866:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177078.476870:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177078.476872:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.476873:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177078.476875:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1786177078.476877:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000008:00000001:1.0:1786177078.476879:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.476881:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177078.476885:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9c00. 00000020:00000040:1.0:1786177078.476887:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177078.476888:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177078.476889:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.476890:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177078.476893:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1786177078.476893:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 02000000:00000001:1.0:1786177078.476896:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177078.476901:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac400. 00000400:00000200:2.0:1786177078.476902:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1786177078.476904:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177078.476908:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00100000:1.0:1786177078.476912:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786177078.476918:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177078.476925:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476931:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177078.476935:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177078.476936:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426539:426539:256:4294967295] 192.168.202.132@tcp LPNI seq info [426501:426501:8:4294967295] 00000100:00000040:1.0:1786177078.476940:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3ea07ed] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177078.476949:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177078.476952:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177078.476955:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177078.476958:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177078.476959:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177078.476961:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177078.476966:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177078.476968:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00080000:1.0:1786177078.476971:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11752 00000400:00000200:1.0:1786177078.476977:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:2.0:1786177078.476981:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177078.476983:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177078.476985:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000010:2.0:1786177078.476987:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00000040:1.0:1786177078.476988:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476993:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177078.476994:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.476998:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:2.0:1786177078.476999:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:1.0:1786177078.477002:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177078.477004:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000800:00000200:2.0:1786177078.477004:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1786177078.477005:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177078.477009:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177078.477012:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000200:2.1:1786177078.477013:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:1.0:1786177078.477015:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000200:2.1:1786177078.477018:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1786177078.477020:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.477023:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:2.0:1786177078.477026:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000040:1.0:1786177078.477028:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000010:2.0:1786177078.477030:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000100:00000001:1.0:1786177078.477031:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177078.477035:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:1.0:1786177078.477037:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177078.477039:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177078.477040:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177078.477041:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1786177078.477042:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1786177078.477043:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.477047:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177078.477049:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000200:2.0:1786177078.477051:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000008:00000001:1.0:1786177078.477053:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.477055:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177078.477058:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000100:00000010:1.0:1786177078.477059:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328ba680. 00000020:00000040:1.0:1786177078.477062:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000400:00000010:2.0:1786177078.477062:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 02000000:00000001:1.0:1786177078.477067:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177078.477067:0:2392:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1786177078.477068:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.477071:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177078.477073:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177078.477075:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177078.477075:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477085:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:1.0:1786177078.477088:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae800. 00000100:00000040:2.0:1786177078.477089:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177078.477096:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.477098:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.477100:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1786177078.477103:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177078.477106:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.477115:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177078.477117:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177078.477119:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177078.477122:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000497a117c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477124:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306429184 00000100:00000001:1.0:1786177078.477132:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177078.477132:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c360. 00000100:00080000:1.0:1786177078.477138:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177078.477140:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177078.477141:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177078.477142:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177078.477144:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177078.477147:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177078.477150:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11752 00000100:00000001:2.0:1786177078.477151:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306429184 00000400:00000200:1.0:1786177078.477155:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000010:2.0:1786177078.477157:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c5a0. 00000400:00000200:1.0:1786177078.477160:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177078.477162:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1786177078.477163:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177078.477167:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1786177078.477168:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177078.477172:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177078.477172:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177078.477176:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177078.477177:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1786177078.477179:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177078.477181:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177078.477182:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177078.477184:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:2.0:1786177078.477185:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177078/real 1786177078] req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177078.477186:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:1.0:1786177078.477191:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:1.0:1786177078.477195:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:1.0:1786177078.477202:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000001:2.0:1786177078.477204:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1786177078.477205:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:2.0:1786177078.477208:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:1.0:1786177078.477209:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:2.0:1786177078.477211:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000100:00000001:1.0:1786177078.477212:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177078.477215:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000020:00000040:1.0:1786177078.477216:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000400:00000010:2.0:1786177078.477219:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00080000:1.0:1786177078.477221:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177078.477223:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1786177078.477225:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177078.477228:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:2.0:1786177078.477229:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177078.477231:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177078.477237:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.477240:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177078.477244:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000200:2.0:1786177078.477246:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1786177078.477247:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.477252:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177078.477258:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb100. 00000100:00000001:2.0:1786177078.477259:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:1.0:1786177078.477262:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000001:2.0:1786177078.477262:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177078.477267:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177078.477268:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177078.477270:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177078.477271:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177078.477276:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1786177078.477276:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177078.477279:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177078.477281:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177078.477286:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae400. 00000100:00000001:2.0:1786177078.477288:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177078.477289:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177078.477295:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177078.477296:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177078.477320:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177078.477325:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177078.477325:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.477329:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177078.477331:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.477331:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.477336:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177078.477339:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:1.0:1786177078.477341:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f57195c8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177078.477342:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177078.477348:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177078.477352:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177078.477385:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177078.477397:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177078.477399:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235572480:192.168.202.132@tcp:250:kworker.0 00000100:00000040:1.0:1786177078.477403:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11763) 00000100:00000001:2.0:1786177078.477405:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.477408:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177078.477411:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8700 x1872930235572480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477416:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177078.477418:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177078.477421:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177078.477423:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177078.477425:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177078.477427:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177078.477428:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177078.477430:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:2.0:1786177078.477434:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177078.477435:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177078.477436:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8700. 00000100:00000001:2.0:1786177078.477439:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177078.477440:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.477441:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.477448:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177078.477449:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.477451:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177078.477452:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177078.477453:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177078.477455:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477460:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177078.477461:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177078.477462:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177078.477464:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177078.477465:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177078.477467:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572608:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177078.477469:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177078.477470:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177078.477471:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177078.477473:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177078.477475:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177078.477476:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177078.477493:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177078.477496:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177078.477500:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177078.477502:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235572608, portal 10 00000100:00000001:2.0:1786177078.477503:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177078.477504:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487985664 : -110371221565952 : ffff9b9e328b8e00) 00000100:00000040:2.0:1786177078.477510:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477514:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177078.477516:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177078.477517:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177078.477519:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235572608, offset 0 00000400:00000200:2.0:1786177078.477521:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177078.477529:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177078.477533:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177078.477535:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177078.477537:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177078.477539:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177078.477540:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177078.477542:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426540:426540:256:4294967295] 192.168.202.132@tcp LPNI seq info [426502:426502:8:4294967295] 00000400:00000200:2.0:1786177078.477545:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177078.477550:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177078.477553:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177078.477555:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177078.477558:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177078.477560:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786177078.477563:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177078.477565:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1786177078.477568:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177078.477569:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177078.477572:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177078.477574:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177078.477576:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177078.477577:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177078.477579:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177078.477581:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477586:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177078.477588:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477592:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.477594:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177078.477596:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177078.477598:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177078.477600:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177078.477601:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306425348 00000100:00000001:2.0:1786177078.477603:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306429184 00000100:00000001:2.0:1786177078.477605:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306425348 00000100:00000001:2.0:1786177078.477607:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306429184 00000100:00000001:2.0:1786177078.477608:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177078.477609:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.477611:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177078.477613:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177078/real 1786177078] req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177078.477618:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177078.477618:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177078.477619:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177078.477621:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177078.477622:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177078.477623:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177078.477625:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177078.477630:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477634:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177078.477635:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.477636:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177078.477637:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177078.477639:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477643:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177078.477646:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477650:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177078.477652:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.477653:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177078.477654:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.477655:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177078.477657:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177078.477659:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177078.477661:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177078.477662:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177078.477664:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177078.477669:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572608:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177078.477672:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177078.477674:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477678:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177078.477679:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177078.477680:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177078.477681:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177078.477682:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177078.477684:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572736:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177078.477686:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177078.477687:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177078.477688:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177078.477689:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177078.477691:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177078.477693:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177078.477695:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177078.477696:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177078.477698:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177078.477700:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235572736, portal 10 00000100:00000001:2.0:1786177078.477701:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177078.477703:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487989248 : -110371221562368 : ffff9b9e328b9c00) 00000100:00000040:2.0:1786177078.477706:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477711:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177078.477712:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177078.477713:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177078.477715:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235572736, offset 0 00000400:00000200:2.0:1786177078.477717:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177078.477719:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177078.477722:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177078.477724:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177078.477725:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177078.477727:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177078.477728:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177078.477730:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426541:426541:256:4294967295] 192.168.202.132@tcp LPNI seq info [426503:426503:8:4294967295] 00000400:00000200:2.0:1786177078.477733:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177078.477737:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177078.477740:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177078.477742:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177078.477744:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177078.477746:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786177078.477749:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177078.477752:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1786177078.477756:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177078.477758:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177078.477761:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177078.477764:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177078.477767:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177078.477769:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177078.477772:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177078.477775:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477791:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177078.477794:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477802:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.477803:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177078.477807:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177078.477810:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177078.477812:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177078.477813:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306425348 00000100:00000001:2.0:1786177078.477815:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306429184 00000100:00000001:2.0:1786177078.477816:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306425348 00000100:00000001:2.0:1786177078.477818:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306429184 00000100:00000001:2.0:1786177078.477819:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177078.477820:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.477822:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177078.477824:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177078/real 1786177078] req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177078.477830:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177078.477831:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177078.477832:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177078.477833:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177078.477834:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177078.477836:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177078.477838:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177078.477842:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477847:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177078.477848:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.477850:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177078.477851:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177078.477853:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477857:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177078.477859:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477863:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177078.477865:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.477866:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177078.477866:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.477868:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177078.477869:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177078.477870:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177078.477872:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177078.477873:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177078.477876:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177078.477880:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572736:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177078.477882:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177078.477884:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477889:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177078.477890:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177078.477891:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177078.477892:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177078.477894:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177078.477896:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572864:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177078.477898:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177078.477899:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177078.477900:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177078.477901:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177078.477904:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177078.477905:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177078.477907:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177078.477909:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177078.477912:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177078.477915:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235572864, portal 4 00000100:00000001:2.0:1786177078.477917:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177078.477919:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487991936 : -110371221559680 : ffff9b9e328ba680) 00000100:00000040:2.0:1786177078.477925:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.477932:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177078.477934:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177078.477936:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177078.477939:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235572864, offset 0 00000400:00000200:2.0:1786177078.477943:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177078.477947:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177078.477951:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177078.477954:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177078.477957:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177078.477959:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177078.477962:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177078.477964:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426542:426542:256:4294967295] 192.168.202.132@tcp LPNI seq info [426504:426504:8:4294967295] 00000400:00000200:2.0:1786177078.477967:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177078.477972:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177078.477975:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177078.477977:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177078.477980:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177078.477982:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786177078.477985:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177078.477987:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1786177078.477989:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177078.477991:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177078.477993:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177078.477995:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177078.477997:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177078.477998:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177078.478000:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177078.478002:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478008:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177078.478010:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478014:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.478015:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177078.478017:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177078.478019:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177078.478021:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177078.478023:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306425348 00000100:00000001:2.0:1786177078.478024:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306429184 00000100:00000001:2.0:1786177078.478026:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306425348 00000100:00000001:2.0:1786177078.478027:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306429184 00000100:00000001:2.0:1786177078.478029:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177078.478030:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.478031:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177078.478033:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177078/real 1786177078] req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177078.478038:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177078.478039:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177078.478040:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177078.478041:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177078.478042:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177078.478044:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177078.478046:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177078.478051:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478054:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177078.478055:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478057:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177078.478058:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177078.478060:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478065:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177078.478067:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478071:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177078.478072:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478073:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177078.478074:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478075:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177078.478076:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177078.478078:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177078.478079:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177078.478081:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177078.478083:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177078.478092:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572864:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177078.478094:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177078.478096:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478100:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177078.478101:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177078.478102:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177078.478103:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177078.478105:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177078.478106:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572992:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177078.478108:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177078.478109:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177078.478110:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177078.478111:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177078.478113:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177078.478115:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177078.478117:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177078.478118:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177078.478120:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177078.478122:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235572992, portal 4 00000100:00000001:2.0:1786177078.478123:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177078.478124:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487994624 : -110371221556992 : ffff9b9e328bb100) 00000100:00000040:2.0:1786177078.478128:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478132:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177078.478134:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177078.478135:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177078.478137:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235572992, offset 0 00000400:00000200:2.0:1786177078.478139:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177078.478141:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177078.478144:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177078.478145:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177078.478147:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177078.478149:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177078.478150:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177078.478152:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426543:426543:256:4294967295] 192.168.202.132@tcp LPNI seq info [426505:426505:8:4294967295] 00000400:00000200:2.0:1786177078.478155:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177078.478159:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177078.478162:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177078.478164:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177078.478166:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177078.478168:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786177078.478170:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177078.478172:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1786177078.478175:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177078.478176:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177078.478178:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177078.478180:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177078.478182:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177078.478183:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177078.478184:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177078.478187:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478191:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177078.478193:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177103 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478197:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.478198:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177078.478200:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177078.478201:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177078.478203:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177078.478205:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306425350 00000100:00000001:2.0:1786177078.478206:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306429184 00000100:00000001:2.0:1786177078.478208:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306425350 00000100:00000001:2.0:1786177078.478209:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306429184 00000100:00000001:2.0:1786177078.478210:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177078.478211:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.478212:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177078.478215:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177078/real 1786177078] req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177078.478219:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177078.478220:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177078.478221:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177078.478222:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177078.478223:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177078.478224:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177078.478226:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177078.478231:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478235:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177078.478236:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478237:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177078.478238:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177078.478240:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478244:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177078.478246:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478250:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177078.478252:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478253:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177078.478253:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478255:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177078.478256:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177078.478257:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177078.478259:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177078.478260:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177078.478263:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177078.478267:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235572992:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177078.478270:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478271:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177078.478273:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8e00 x1872930235572608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478277:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177078.478278:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177078.478279:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177078.478280:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177078.478281:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177078.478282:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177078.478283:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177078.478284:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:2.0:1786177078.478286:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177078.478287:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177078.478288:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8e00. 00000100:00000001:2.0:1786177078.478290:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177078.478290:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478292:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177078.478294:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9c00 x1872930235572736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478302:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177078.478303:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177078.478304:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177078.478305:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177078.478306:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177078.478306:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177078.478308:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177078.478308:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:2.0:1786177078.478309:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177078.478310:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177078.478311:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9c00. 00000100:00000001:2.0:1786177078.478312:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177078.478313:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478314:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177078.478316:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328ba680 x1872930235572864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478319:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177078.478320:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177078.478321:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177078.478322:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177078.478323:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177078.478324:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177078.478325:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177078.478325:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:2.0:1786177078.478327:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177078.478327:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177078.478328:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328ba680. 00000100:00000001:2.0:1786177078.478331:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177078.478331:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478332:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177078.478334:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb100 x1872930235572992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177103 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177078.478338:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177078.478338:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177078.478340:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177078.478341:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177078.478342:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177078.478343:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177078.478344:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177078.478345:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:2.0:1786177078.478346:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177078.478347:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177078.478348:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb100. 00000100:00000001:2.0:1786177078.478350:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177078.478351:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478352:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177078.478354:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177078.478355:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177078.478371:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177078.478373:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177078.988425:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177078.988434:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177078.988437:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177078.988444:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177078.988449:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177078.988454:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177078.988457:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177078.988459:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177078.988464:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177079.372494:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177079.372504:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177079.372506:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177079.372512:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177079.372515:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177079.372516:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177079.372521:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177079.372522:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177079.372525:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177079.372529:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177079.372530:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177079.372532:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177079.372534:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177079.372540:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177079.372541:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177079.372631:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177079.372639:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177079.372644:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177080.012449:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177080.012457:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177080.012461:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177080.012478:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177080.012481:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177080.012484:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177080.012486:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177080.012488:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177080.012490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177080.396496:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177080.396500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177080.396503:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177080.396504:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177080.396506:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177080.396509:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1786177080.396510:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177080.396512:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177080.396514:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1786177080.396515:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177080.396518:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177080.396518:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177080.396521:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1786177080.396521:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177080.396524:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177080.396524:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177080.396526:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177080.396527:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177080.396528:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177080.396540:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177080.396542:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177080.396612:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177080.396616:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177080.396620:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177081.036424:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177081.036429:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177081.036431:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177081.036436:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177081.036441:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177081.036444:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177081.036446:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177081.036449:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177081.036451:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177081.420427:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177081.420435:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177081.420435:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177081.420438:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177081.420439:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177081.420444:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177081.420445:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177081.420446:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177081.420448:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177081.420461:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177081.420463:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177081.420465:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177081.420467:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177081.420471:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177081.420474:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177081.420515:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177081.420519:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177081.420521:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786177081.996493:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786177081.996502:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786177081.996508:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1786177081.996664:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1786177081.996665:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1786177081.996681:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786177081.996688:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786177081.996692:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177082.060430:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177082.060439:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177082.060442:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177082.060460:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177082.060464:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177082.060468:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177082.060471:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177082.060473:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177082.060477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177082.380436:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306429184, at 4306429248 00000100:00000040:0.0:1786177082.380437:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306429184, at 4306429248 00000100:00000010:0.0:1786177082.380465:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c360. 00000100:00000010:1.0:1786177082.380468:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c5a0. 00000100:00000001:3.0:1786177082.444491:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177082.444495:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177082.444499:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177082.444500:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1786177082.444501:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177082.444502:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177082.444506:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177082.444506:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177082.444510:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1786177082.444510:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177082.444512:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177082.444513:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177082.444514:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177082.444516:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177082.444517:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177082.444519:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177082.444521:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177082.444523:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177082.444523:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177082.444526:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177082.444528:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177082.444529:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177082.444531:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177082.444532:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177082.444534:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177082.444536:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177082.444537:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177082.444600:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177082.444604:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177082.444608:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177083.084526:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177083.084533:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177083.084535:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177083.084540:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177083.084544:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177083.084550:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177083.084553:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177083.084557:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177083.084560:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177083.468480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177083.468482:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177083.468486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177083.468494:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177083.468496:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177083.468496:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177083.468499:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177083.468500:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177083.468503:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177083.468508:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177083.468514:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177083.468520:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177083.468524:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177083.468528:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177083.468532:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177083.468575:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177083.468580:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177083.468585:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177083.596493:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177083.596505:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177083.596507:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177083.596512:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177083.596526:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177083.596534:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11758 00000400:00000200:2.0:1786177083.596544:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177083.596551:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177083.596555:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177083.596558:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177083.596562:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177083.596565:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177083.596568:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177083.596572:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177083.596574:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177083.596577:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177083.596580:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177083.596583:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177083.596586:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177083.596589:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177083.596591:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177083.596593:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177083.596595:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177083.596599:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177083.596611:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177083.596614:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177083.596617:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177083.596621:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177083.596624:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177083.596626:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177083.596629:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177083.596640:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000020:00000040:2.0:1786177083.596645:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177083.596649:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177083.596651:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177083.596655:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177083.596658:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177083.596664:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177083.596671:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177083.596678:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177083.596689:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177083.596707:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177083.596711:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177083.596717:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177083.596741:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177083.596745:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177083.596748:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177083.596750:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177083.596753:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177083.596757:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177083.596761:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11758 00000100:00000001:1.0:1786177083.596766:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1786177083.596766:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177083.596771:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1786177083.596774:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177083.596775:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177083.596781:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177083.596788:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177083.596794:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177083.596798:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177083.596799:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177083.596802:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1786177083.596804:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177083.596805:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177083.596810:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177083.596811:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00100000:1.0:1786177083.596815:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235573120:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177083.596817:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177083.596822:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1786177083.596823:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177083.596826:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 02000000:00000001:1.0:1786177083.596827:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177083.596830:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177083.596831:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1786177083.596833:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:2.0:1786177083.596836:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 02000000:00000010:1.0:1786177083.596839:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae800. 00000020:00000001:2.0:1786177083.596841:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 02000000:00000001:1.0:1786177083.596842:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177083.596846:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000400:00000200:1.0:1786177083.596852:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177083.596854:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177083.596858:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1786177083.596859:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00000040:2.0:1786177083.596861:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177083.596865:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:1.0:1786177083.596869:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000020:00000040:2.0:1786177083.596870:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177083.596876:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000200:1.0:1786177083.596880:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235573120, portal 25 00000100:00000001:2.0:1786177083.596880:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177083.596885:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177083.596887:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177083.596890:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469625472 : -110371239926144 : ffff9b9e31736680) 02000000:00000001:2.0:1786177083.596892:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177083.596898:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177083.596899:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177083.596904:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177083.596909:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:2.0:1786177083.596910:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1786177083.596911:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177083.596916:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000008:00000001:2.0:1786177083.596917:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177083.596919:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235573120, offset 0 00000100:00000001:2.0:1786177083.596921:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177083.596926:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000010:2.0:1786177083.596927:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 00000020:00000040:2.0:1786177083.596932:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177083.596935:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177083.596936:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177083.596940:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177083.596942:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:2.0:1786177083.596944:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1786177083.596948:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1786177083.596948:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177083.596952:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177083.596954:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177083.596959:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 02000000:00000010:2.0:1786177083.596960:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000400:00000200:1.0:1786177083.596962:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177083.596968:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426544:426544:256:4294967295] 192.168.202.132@tcp LPNI seq info [426506:426506:8:4294967295] 00000100:00000001:2.0:1786177083.596969:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177083.596976:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:2.0:1786177083.596981:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177083.596996:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177083.596998:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177083.597003:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177083.597008:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000040:2.0:1786177083.597011:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1786177083.597016:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000001:2.0:1786177083.597027:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177083.597031:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177083.597037:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.0:1786177083.597038:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177083.597040:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177083.597045:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1786177083.597046:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1786177083.597049:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177083.597057:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11758 00000800:00000200:1.1:1786177083.597060:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:2.0:1786177083.597066:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:1.1:1786177083.597066:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1786177083.597075:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177083.597075:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000010:1.0:1786177083.597079:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00080000:2.0:1786177083.597081:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177083.597085:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1786177083.597088:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177083.597093:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1786177083.597094:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177083.597098:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000100:00000040:2.0:1786177083.597099:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177083.597101:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:2.0:1786177083.597105:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177083.597107:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1786177083.597111:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000200:1.0:1786177083.597113:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177083.597115:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177083.597119:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177083.597123:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000100:00000001:1.0:1786177083.597125:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1786177083.597126:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177083.597130:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177083.597132:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177083.597141:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177083.597142:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.597146:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177083.597146:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177083.597151:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177083.597153:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177083.597155:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177083.597160:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:2.0:1786177083.597161:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00000001:1.0:1786177083.597163:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306434304 00000100:00080000:2.0:1786177083.597168:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000010:1.0:1786177083.597170:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62540. 00000100:00000001:1.0:1786177083.597176:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177083.597183:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306434304 00000100:00000001:2.0:1786177083.597183:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177083.597189:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:1.0:1786177083.597190:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c626c0. 00000100:00000001:1.0:1786177083.597194:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1786177083.597194:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177083.597199:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177083.597200:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.597202:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177083.597204:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177083.597207:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000008:00100000:2.0:1786177083.597209:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177083.597214:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177083.597215:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177083/real 1786177083] req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177083.597217:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177083.597223:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 00000020:00000040:2.0:1786177083.597228:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000001:1.0:1786177083.597232:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1786177083.597232:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1786177083.597237:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1786177083.597237:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177083.597243:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000100:00000001:2.0:1786177083.597243:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177083.597246:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1786177083.597249:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177083.597252:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177083.597255:0:2392:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1786177083.597257:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177083.597261:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1786177083.597264:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:2.0:1786177083.597270:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177083.597275:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177083.597280:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177083.597282:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177083.597285:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.597289:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177083.597292:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177083.597297:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1786177083.597300:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177083.597302:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.597310:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177083.597312:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177083.597317:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.597338:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177083.597338:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.597345:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1786177083.597345:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177083.597346:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177083.597349:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.597352:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1786177083.597354:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177083.597355:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177083.597377:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177083.597379:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177083.597381:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177083.597385:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177083.597391:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235573120:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177083.597397:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177083.597399:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177083.597401:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1786177083.597403:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1786177083.597405:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736680 x1872930235573120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177083.597410:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177083.597413:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:2.0:1786177083.597417:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11758 02000000:00000001:1.0:1786177083.597418:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177083.597421:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177083.597425:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1786177083.597426:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:1.0:1786177083.597428:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177083.597444:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177083.597447:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000200:2.0:1786177083.597450:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 02000000:00000010:1.0:1786177083.597451:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 00000100:00080000:2.0:1786177083.597456:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1786177083.597457:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177083.597461:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177083.597464:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736680. 00000100:00000040:2.0:1786177083.597465:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177083.597467:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177083.597469:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177083.597471:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177083.597472:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177083.597478:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177083.597480:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177083.597482:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177083.597483:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177083.597487:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177083.597488:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1786177083.597491:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:1.0:1786177083.597492:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177083.597494:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1786177083.597496:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1786177083.597500:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177083.597501:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177083.597507:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177083.597512:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177083.597517:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1786177083.597521:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177083.597522:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177083.597526:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177083.597528:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1786177083.597530:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177083.597532:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177083.597536:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:1.0:1786177083.597538:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177083.597542:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00100000:1.0:1786177083.597547:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235573248:192.168.202.132@tcp:38:kworker.0 00000100:00080000:2.0:1786177083.597550:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177083.597557:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1786177083.597557:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177083.597562:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177083.597562:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177083.597567:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177083.597569:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177083.597572:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1786177083.597574:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177083.597578:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae800. 00000100:00000001:2.0:1786177083.597578:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177083.597581:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 02000000:00000001:1.0:1786177083.597583:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1786177083.597586:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177083.597589:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177083.597593:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000010:2.0:1786177083.597596:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000400:00000010:1.0:1786177083.597599:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000020:00000040:2.0:1786177083.597601:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000400:00000010:1.0:1786177083.597605:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 02000000:00000001:2.0:1786177083.597606:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177083.597610:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177083.597611:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235573248, portal 10 00000100:00000001:2.0:1786177083.597613:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177083.597614:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177083.597618:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469626368 : -110371239925248 : ffff9b9e31736a00) 02000000:00000001:2.0:1786177083.597619:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177083.597623:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177083.597626:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1786177083.597630:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000001:1.0:1786177083.597634:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1786177083.597637:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177083.597638:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177083.597642:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177083.597645:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235573248, offset 0 00000100:00100000:2.0:1786177083.597649:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177083.597650:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177083.597658:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177083.597665:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:2.0:1786177083.597669:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177083.597670:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1786177083.597674:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177083.597675:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177083.597679:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177083.597684:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000040:2.0:1786177083.597686:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177083.597688:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426545:426545:256:4294967295] 192.168.202.132@tcp LPNI seq info [426507:426507:8:4294967295] 00000400:00000200:1.0:1786177083.597699:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177083.597710:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177083.597712:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177083.597716:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00080000:2.0:1786177083.597718:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1786177083.597720:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00080000:2.0:1786177083.597727:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000200:1.0:1786177083.597728:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177083.597732:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.0:1786177083.597733:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00080000:2.0:1786177083.597735:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1786177083.597740:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177083.597742:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1786177083.597743:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00080000:2.0:1786177083.597748:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11758 00000800:00000100:1.0:1786177083.597753:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177083.597755:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:2.0:1786177083.597758:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177083.597759:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177083.597767:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177083.597768:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177083.597769:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177083.597773:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00080000:2.0:1786177083.597773:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177083.597775:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1786177083.597782:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000200:1.0:1786177083.597783:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177083.597789:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177083.597795:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177083.597795:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177083.597802:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177083.597802:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177083.597808:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177083.597812:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000001:1.0:1786177083.597813:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177083.597817:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177083.597818:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177083.597821:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177083.597823:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000040:1.0:1786177083.597825:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000020:00000001:2.0:1786177083.597828:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000200:1.0:1786177083.597829:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177083.597834:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306430468 00000020:00000001:2.0:1786177083.597834:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177083.597838:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306434304 00000100:00000001:1.0:1786177083.597842:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306430468 00000100:00000040:2.0:1786177083.597843:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177083.597845:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306434304 00000100:00000001:2.0:1786177083.597849:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177083.597852:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177083.597855:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177083.597858:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:1.0:1786177083.597861:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1786177083.597863:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177083.597867:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.597870:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177083.597871:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177083.597875:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1786177083.597877:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1786177083.597884:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177083/real 1786177083] req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177083.597884:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177083.597888:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177083.597892:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177083.597897:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177083.597903:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1786177083.597904:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177083.597906:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177083.597908:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000100:00000010:2.0:1786177083.597908:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000400:00000200:1.0:1786177083.597913:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000020:00000040:2.0:1786177083.597915:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000400:00000010:1.0:1786177083.597916:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 02000000:00000001:2.0:1786177083.597921:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177083.597922:0:2392:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1786177083.597926:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177083.597928:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177083.597932:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177083.597935:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177083.597939:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177083.597940:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1786177083.597946:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:1.0:1786177083.597947:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177083.597950:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.597953:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177083.597954:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177083.597954:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177083.597961:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177083.597966:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177083.597968:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177083.597976:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177083.597986:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177083.597987:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177083.597991:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177083.597992:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.597994:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177083.597996:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.597997:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177083.598001:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1786177083.598001:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177083.598006:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177083.598012:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177083.598016:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177083.598024:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177083.598025:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177083.598033:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11768) 00000100:00100000:1.0:1786177083.598034:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235573248:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177083.598040:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177083.598044:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598053:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177083.598054:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177083.598056:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177083.598058:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177083.598059:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177083.598061:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235573376:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177083.598064:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177083.598065:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177083.598066:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177083.598067:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177083.598070:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177083.598071:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177083.598075:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177083.598078:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177083.598081:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177083.598084:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235573376, portal 10 00000100:00000001:1.0:1786177083.598086:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177083.598087:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469615616 : -110371239936000 : ffff9b9e31734000) 00000100:00000040:1.0:1786177083.598093:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598100:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177083.598102:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177083.598105:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177083.598108:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235573376, offset 0 00000400:00000200:1.0:1786177083.598112:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177083.598115:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177083.598118:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177083.598121:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177083.598122:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177083.598124:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177083.598126:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177083.598128:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426546:426546:256:4294967295] 192.168.202.132@tcp LPNI seq info [426508:426508:8:4294967295] 00000400:00000200:1.0:1786177083.598131:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177083.598136:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177083.598139:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177083.598141:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177083.598144:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177083.598146:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786177083.598149:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177083.598151:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1786177083.598153:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177083.598154:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177083.598157:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177083.598159:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177083.598161:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177083.598162:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177083.598164:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177083.598166:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598172:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177083.598174:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598178:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.598179:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177083.598181:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177083.598184:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177083.598186:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177083.598187:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306430468 00000100:00000001:1.0:1786177083.598189:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306434304 00000100:00000001:1.0:1786177083.598190:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306430468 00000100:00000001:1.0:1786177083.598192:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306434304 00000100:00000001:1.0:1786177083.598193:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177083.598195:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.598196:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177083.598199:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177083/real 1786177083] req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177083.598204:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177083.598205:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177083.598207:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177083.598209:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177083.598210:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177083.598213:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177083.598216:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177083.598225:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598232:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177083.598234:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598235:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177083.598237:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177083.598241:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598249:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177083.598253:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598260:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177083.598262:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598263:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177083.598264:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598266:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177083.598267:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177083.598269:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177083.598272:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177083.598273:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177083.598276:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177083.598281:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235573376:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177083.598284:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598285:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177083.598287:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736a00 x1872930235573248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598291:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177083.598292:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177083.598294:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177083.598296:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177083.598297:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177083.598298:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177083.598299:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177083.598300:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:1.0:1786177083.598302:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177083.598303:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177083.598304:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736a00. 00000100:00000001:1.0:1786177083.598306:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177083.598307:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598308:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177083.598310:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734000 x1872930235573376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598314:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177083.598315:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177083.598316:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177083.598317:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177083.598318:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177083.598319:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177083.598320:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177083.598321:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:1.0:1786177083.598323:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177083.598324:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177083.598325:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734000. 00000100:00000001:1.0:1786177083.598326:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177083.598327:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598328:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598331:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177083.598331:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.598333:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177083.598334:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177083.598335:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177083.598337:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598342:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177083.598343:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177083.598344:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177083.598345:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177083.598347:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177083.598349:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235573504:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177083.598351:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177083.598352:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177083.598353:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177083.598354:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177083.598371:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177083.598372:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177083.598375:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177083.598377:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177083.598380:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177083.598382:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235573504, portal 4 00000100:00000001:1.0:1786177083.598383:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177083.598384:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469624576 : -110371239927040 : ffff9b9e31736300) 00000100:00000040:1.0:1786177083.598388:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598397:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177083.598399:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177083.598400:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177083.598402:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235573504, offset 0 00000400:00000200:1.0:1786177083.598404:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177083.598407:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177083.598409:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177083.598411:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177083.598413:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177083.598415:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177083.598416:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177083.598418:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426547:426547:256:4294967295] 192.168.202.132@tcp LPNI seq info [426509:426509:8:4294967295] 00000400:00000200:1.0:1786177083.598421:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177083.598425:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177083.598442:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177083.598445:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177083.598448:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177083.598450:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786177083.598453:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177083.598455:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1786177083.598458:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177083.598459:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177083.598461:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177083.598464:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177083.598466:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177083.598467:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177083.598469:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177083.598472:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598477:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177083.598479:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598487:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.598488:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177083.598491:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177083.598493:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177083.598495:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177083.598497:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306430469 00000100:00000001:1.0:1786177083.598499:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306434304 00000100:00000001:1.0:1786177083.598500:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306430469 00000100:00000001:1.0:1786177083.598502:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306434304 00000100:00000001:1.0:1786177083.598503:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177083.598505:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.598507:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177083.598511:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177083/real 1786177083] req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177083.598519:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177083.598521:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177083.598522:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177083.598525:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177083.598526:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177083.598528:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177083.598532:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177083.598541:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598548:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177083.598550:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598552:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177083.598553:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177083.598558:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598566:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177083.598570:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598577:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177083.598580:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598582:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177083.598583:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598585:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177083.598587:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177083.598590:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177083.598594:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177083.598597:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177083.598602:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177083.598609:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235573504:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177083.598613:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177083.598615:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598620:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177083.598621:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177083.598623:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177083.598624:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177083.598625:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177083.598627:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235573632:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177083.598629:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177083.598630:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177083.598632:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177083.598633:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177083.598635:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177083.598637:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177083.598639:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177083.598641:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177083.598643:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177083.598645:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235573632, portal 4 00000100:00000001:1.0:1786177083.598646:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177083.598647:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469616512 : -110371239935104 : ffff9b9e31734380) 00000100:00000040:1.0:1786177083.598651:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598655:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177083.598657:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177083.598658:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177083.598660:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235573632, offset 0 00000400:00000200:1.0:1786177083.598662:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177083.598665:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177083.598667:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177083.598669:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177083.598671:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177083.598672:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177083.598674:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177083.598676:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426548:426548:256:4294967295] 192.168.202.132@tcp LPNI seq info [426510:426510:8:4294967295] 00000400:00000200:1.0:1786177083.598679:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177083.598683:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177083.598687:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177083.598689:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177083.598691:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177083.598693:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786177083.598696:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177083.598698:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1786177083.598700:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177083.598702:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177083.598705:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177083.598707:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177083.598709:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177083.598710:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177083.598711:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177083.598713:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598719:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177083.598720:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177108 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598725:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.598726:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177083.598728:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177083.598729:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177083.598731:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177083.598732:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306430471 00000100:00000001:1.0:1786177083.598734:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306434304 00000100:00000001:1.0:1786177083.598735:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306430471 00000100:00000001:1.0:1786177083.598737:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306434304 00000100:00000001:1.0:1786177083.598738:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177083.598739:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.598740:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177083.598743:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177083/real 1786177083] req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177083.598747:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177083.598748:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177083.598749:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177083.598751:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177083.598752:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177083.598754:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177083.598756:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177083.598761:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598765:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177083.598765:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598766:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177083.598767:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177083.598769:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598773:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177083.598776:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598779:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177083.598781:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598782:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177083.598783:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598784:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177083.598785:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177083.598786:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177083.598788:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177083.598790:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177083.598792:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177083.598797:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235573632:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177083.598799:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598800:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177083.598803:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736300 x1872930235573504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598807:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177083.598808:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177083.598810:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177083.598811:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177083.598812:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177083.598813:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177083.598814:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177083.598815:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:1.0:1786177083.598817:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177083.598818:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177083.598819:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736300. 00000100:00000001:1.0:1786177083.598820:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177083.598822:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598823:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177083.598825:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734380 x1872930235573632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177108 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177083.598829:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177083.598830:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177083.598830:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177083.598832:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177083.598832:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177083.598833:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177083.598834:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177083.598835:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:1.0:1786177083.598836:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177083.598837:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177083.598837:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734380. 00000100:00000001:1.0:1786177083.598839:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177083.598839:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598840:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177083.598843:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177083.598844:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177083.598846:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177083.598847:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177084.108499:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177084.108508:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177084.108511:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177084.108518:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177084.108523:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177084.108528:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177084.108532:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177084.108535:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177084.108538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177084.492459:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177084.492466:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177084.492468:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177084.492472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177084.492473:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177084.492476:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177084.492478:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177084.492481:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177084.492483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177084.492483:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177084.492489:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177084.492490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177084.492492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177084.492493:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177084.492496:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177084.492536:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177084.492539:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177084.492542:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177085.132500:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177085.132509:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177085.132510:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177085.132516:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177085.132520:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177085.132523:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177085.132525:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177085.132527:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177085.132530:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177085.516477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177085.516482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177085.516488:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177085.516490:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177085.516496:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177085.516496:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177085.516500:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177085.516504:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177085.516507:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177085.516507:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177085.516511:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177085.516512:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177085.516518:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177085.516521:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177085.516524:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177085.516581:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177085.516585:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177085.516588:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177086.156893:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177086.156902:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177086.156904:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177086.156911:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177086.156916:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177086.156920:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177086.156922:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177086.156924:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177086.156927:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177086.540528:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177086.540534:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177086.540537:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177086.540539:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177086.540540:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177086.540548:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177086.540548:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177086.540551:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177086.540553:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177086.540556:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177086.540558:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177086.540559:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177086.540562:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177086.540566:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177086.540569:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177086.540624:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177086.540629:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177086.540632:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177087.180841:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177087.180852:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177087.180855:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177087.180862:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177087.180868:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177087.180872:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177087.180875:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177087.180879:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177087.180882:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177087.500486:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306434304, at 4306434368 00000100:00000040:1.0:1786177087.500486:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306434304, at 4306434368 00000100:00000010:0.0:1786177087.500503:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62540. 00000100:00000010:1.0:1786177087.500505:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c626c0. 00000100:00000001:1.0:1786177087.564477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177087.564479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177087.564485:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177087.564487:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177087.564490:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177087.564491:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177087.564495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177087.564497:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177087.564498:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177087.564501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177087.564502:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177087.564506:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177087.564509:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177087.564511:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177087.564513:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177087.564567:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177087.564572:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177087.564576:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177088.204588:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177088.204603:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177088.204606:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177088.204616:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177088.204621:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177088.204628:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.204632:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177088.204647:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.204652:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.588485:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177088.588487:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177088.588491:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.588492:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177088.588495:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177088.588498:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177088.588500:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177088.588501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.588506:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177088.588508:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177088.588510:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177088.588511:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.588515:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177088.588518:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177088.588520:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177088.588625:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177088.588631:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177088.588634:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177088.716549:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177088.716558:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177088.716560:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177088.716563:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177088.716566:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177088.716573:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11763 00000400:00000200:1.0:1786177088.716580:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177088.716586:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177088.716588:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177088.716591:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177088.716594:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177088.716596:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177088.716598:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177088.716601:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177088.716603:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177088.716605:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177088.716608:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177088.716610:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177088.716612:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177088.716616:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177088.716618:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177088.716619:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177088.716623:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177088.716626:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177088.716629:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177088.716632:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177088.716635:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177088.716638:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177088.716642:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.716644:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177088.716646:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177088.716656:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb480. 00000020:00000040:1.0:1786177088.716660:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177088.716670:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177088.716672:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.716675:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177088.716677:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177088.716681:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177088.716688:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae800. 00000100:00000001:1.0:1786177088.716693:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177088.716701:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177088.716710:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177088.716713:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177088.716717:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b192a357] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177088.716736:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177088.716740:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177088.716744:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177088.716745:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177088.716747:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177088.716751:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177088.716755:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11763 00000400:00000200:1.0:1786177088.716761:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177088.716765:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177088.716768:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177088.716769:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1786177088.716772:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177088.716775:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177088.716777:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177088.716780:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177088.716783:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177088.716787:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:2.0:1786177088.716792:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177088.716793:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177088.716805:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177088.716808:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177088.716811:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177088.716816:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177088.716819:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1786177088.716822:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177088.716823:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.716831:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1786177088.716832:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:2.0:1786177088.716835:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177088.716837:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1786177088.716842:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177088.716843:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177088.716848:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00100000:2.0:1786177088.716850:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235573760:192.168.202.132@tcp:250:kworker.0 00000020:00000040:1.0:1786177088.716854:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:2.0:1786177088.716858:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1786177088.716863:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:2.0:1786177088.716864:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1786177088.716869:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.716869:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177088.716876:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177088.716877:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1786177088.716884:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:2.0:1786177088.716885:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa400. 02000000:00000001:1.0:1786177088.716890:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.716891:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.716896:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:2.0:1786177088.716901:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000008:00100000:1.0:1786177088.716902:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000010:2.0:1786177088.716907:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000008:00000001:1.0:1786177088.716908:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.716914:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177088.716917:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000010:1.0:1786177088.716919:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9f80. 00000020:00000040:1.0:1786177088.716924:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177088.716927:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:2.0:1786177088.716927:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235573760, portal 25 02000000:00000001:1.0:1786177088.716931:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.716933:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177088.716935:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1786177088.716938:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487995520 : -110371221556096 : ffff9b9e328bb480) 02000000:00000001:1.0:1786177088.716939:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177088.716943:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177088.716950:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac400. 00000100:00000040:2.0:1786177088.716950:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177088.716958:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.716966:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1786177088.716969:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1786177088.716974:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177088.716980:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000001:1.0:1786177088.716987:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1786177088.716987:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235573760, offset 0 00000100:00000001:1.0:1786177088.716993:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177088.717004:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4f3e5a2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177088.717004:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177088.717018:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177088.717022:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177088.717027:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1786177088.717028:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00080000:1.0:1786177088.717036:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:2.0:1786177088.717038:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1786177088.717040:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177088.717044:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1786177088.717046:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:1.0:1786177088.717048:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177088.717053:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1786177088.717057:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11763 00000400:00000200:2.0:1786177088.717060:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177088.717070:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177088.717071:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426549:426549:256:4294967295] 192.168.202.132@tcp LPNI seq info [426511:426511:8:4294967295] 00000400:00000200:1.0:1786177088.717080:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:2.0:1786177088.717081:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177088.717087:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177088.717092:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177088.717098:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177088.717103:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000040:1.0:1786177088.717104:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177088.717110:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177088.717115:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177088.717119:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000800:00000200:2.0:1786177088.717119:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000020:00000001:1.0:1786177088.717124:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177088.717128:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000800:00000010:2.0:1786177088.717129:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000020:00000001:1.0:1786177088.717133:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177088.717139:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000200:2.0:1786177088.717141:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:1.0:1786177088.717147:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:2.0:1786177088.717148:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1786177088.717153:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1786177088.717159:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:1.0:1786177088.717160:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:2.1:1786177088.717165:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1786177088.717166:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177088.717173:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000100:2.0:1786177088.717177:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177088.717180:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177088.717186:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:1.0:1786177088.717191:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:2.0:1786177088.717197:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1786177088.717198:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177088.717204:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1786177088.717204:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 02000000:00000001:1.0:1786177088.717208:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1786177088.717209:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 02000000:00000001:1.0:1786177088.717216:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.717218:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1786177088.717220:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177088.717223:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000200:2.0:1786177088.717225:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1786177088.717228:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.717234:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177088.717242:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb800. 00000020:00000040:1.0:1786177088.717247:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000001:2.0:1786177088.717248:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1786177088.717252:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177088.717256:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177088.717257:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177088.717261:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177088.717264:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177088.717266:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177088.717271:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac800. 00000100:00000001:1.0:1786177088.717278:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.717283:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.717286:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1786177088.717289:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177088.717294:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177088.717303:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177088.717306:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177088.717309:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306439424 00000100:00000001:1.0:1786177088.717310:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177088.717316:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c960. 00000100:00000040:1.0:1786177088.717319:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008148c2a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.717325:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177088.717333:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.717336:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306439424 00000100:00080000:1.0:1786177088.717338:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:2.0:1786177088.717341:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c420. 00000100:00080000:1.0:1786177088.717345:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177088.717347:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177088.717348:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177088.717353:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177088.717353:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177088.717399:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.717405:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786177088.717406:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1786177088.717413:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11763 00000100:00100000:2.0:1786177088.717417:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177088/real 1786177088] req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177088.717423:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177088.717433:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177088.717439:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000001:2.0:1786177088.717441:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177088.717445:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:1.0:1786177088.717446:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:2.0:1786177088.717449:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000100:00000001:1.0:1786177088.717451:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177088.717455:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000100:00000040:1.0:1786177088.717456:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:2.0:1786177088.717460:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:1.0:1786177088.717463:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177088.717465:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1786177088.717470:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000200:2.0:1786177088.717473:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177088.717474:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177088.717478:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177088.717482:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177088.717489:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000200:2.0:1786177088.717494:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177088.717495:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:1.0:1786177088.717506:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177088.717512:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177088.717512:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177088.717516:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177088.717518:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177088.717520:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177088.717525:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177088.717525:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177088.717532:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00100000:2.0:1786177088.717535:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177088.717539:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177088.717546:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177088.717551:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177088.717554:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177088.717556:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177088.717563:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177088.717563:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177088.717568:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177088.717572:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177088.717580:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.717582:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177088.717584:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.717590:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1786177088.717592:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9180. 00000100:00000001:2.0:1786177088.717593:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177088.717596:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177088.717599:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:2.0:1786177088.717600:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177088.717605:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1786177088.717606:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177088.717609:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177088.717611:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1786177088.717614:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177088.717617:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:2.0:1786177088.717618:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 02000000:00000001:1.0:1786177088.717621:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177088.717628:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac000. 00000100:00000001:2.0:1786177088.717632:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1786177088.717633:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177088.717643:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177088.717645:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177088.717657:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177088.717662:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177088.717670:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d01dc3b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177088.717673:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235573760:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177088.717686:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177088.717689:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.717690:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1786177088.717695:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1786177088.717697:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb480 x1872930235573760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177088.717703:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177088.717707:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177088.717711:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177088.717714:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1786177088.717716:0:40394:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1786177088.717719:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1786177088.717723:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11763 02000000:00000010:2.0:1786177088.717725:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa400. 00000400:00000200:1.0:1786177088.717731:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177088.717732:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177088.717736:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:1.0:1786177088.717737:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000020:00000040:2.0:1786177088.717739:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00080000:1.0:1786177088.717743:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000020:00000001:2.0:1786177088.717744:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177088.717749:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae800. 00000100:00000040:1.0:1786177088.717751:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:2.0:1786177088.717755:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1786177088.717756:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1786177088.717758:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:1.0:1786177088.717762:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000010:2.0:1786177088.717763:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb480. 00000100:00000001:1.0:1786177088.717770:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177088.717770:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177088.717774:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1786177088.717777:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177088.717778:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177088.717781:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177088.717785:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1786177088.717787:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:1.0:1786177088.717788:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:2.0:1786177088.717792:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177088.717793:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177088.717797:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1786177088.717798:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177088.717800:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177088.717804:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177088.717807:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:2.0:1786177088.717812:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177088.717813:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177088.717818:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177088.717822:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177088.717827:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:1.0:1786177088.717832:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177088.717837:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177088.717841:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177088.717841:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1786177088.717846:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177088.717846:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177088.717851:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.717851:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1786177088.717855:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:2.0:1786177088.717856:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:1.0:1786177088.717857:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177088.717863:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.717863:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.717868:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177088.717872:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8a80. 00000100:00100000:2.0:1786177088.717873:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235573888:192.168.202.132@tcp:38:kworker.0 00000020:00000040:1.0:1786177088.717874:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177088.717879:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177088.717882:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177088.717885:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1786177088.717885:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177088.717889:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177088.717890:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177088.717891:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.717894:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177088.717896:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:2.0:1786177088.717898:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1786177088.717901:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177088.717904:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa400. 00000100:00100000:1.0:1786177088.717908:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1786177088.717910:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177088.717920:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177088.717926:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00000001:1.0:1786177088.717931:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177088.717934:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177088.717934:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000040:1.0:1786177088.717942:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002bb61190] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177088.717942:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235573888, portal 10 00000100:00000001:2.0:1786177088.717949:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177088.717952:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487990144 : -110371221561472 : ffff9b9e328b9f80) 00000100:00000001:1.0:1786177088.717955:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177088.717960:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11773) 00000100:00000040:2.0:1786177088.717964:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.717974:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177088.717978:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177088.717981:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177088.717984:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235573888, offset 0 00000400:00000200:2.0:1786177088.717988:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177088.717993:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177088.717997:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177088.718000:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177088.718003:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177088.718006:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177088.718009:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177088.718012:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426550:426550:256:4294967295] 192.168.202.132@tcp LPNI seq info [426512:426512:8:4294967295] 00000400:00000200:2.0:1786177088.718017:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177088.718025:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177088.718030:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177088.718034:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177088.718038:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177088.718041:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786177088.718046:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177088.718049:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1786177088.718053:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177088.718055:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177088.718058:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177088.718062:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177088.718065:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177088.718067:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177088.718070:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177088.718074:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718090:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177088.718093:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718101:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.718103:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177088.718106:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177088.718109:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177088.718112:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177088.718114:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306435588 00000100:00000001:2.0:1786177088.718117:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306439424 00000100:00000001:2.0:1786177088.718120:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306435588 00000100:00000001:2.0:1786177088.718122:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306439424 00000100:00000001:2.0:1786177088.718124:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177088.718126:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.718129:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177088.718132:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177088/real 1786177088] req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177088.718141:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177088.718143:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177088.718145:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177088.718148:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177088.718150:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177088.718152:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177088.718156:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177088.718165:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718173:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177088.718174:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718177:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177088.718178:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177088.718182:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718190:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177088.718194:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718202:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177088.718204:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718206:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177088.718207:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718209:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177088.718211:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177088.718214:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177088.718217:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177088.718220:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177088.718224:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177088.718233:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235573888:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177088.718236:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177088.718240:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718248:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177088.718249:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.718252:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177088.718253:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177088.718256:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177088.718259:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574016:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177088.718263:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177088.718265:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177088.718267:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.718269:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177088.718272:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786177088.718274:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177088.718278:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177088.718280:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177088.718283:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177088.718286:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235574016, portal 10 00000100:00000001:2.0:1786177088.718288:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177088.718290:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487996416 : -110371221555200 : ffff9b9e328bb800) 00000100:00000040:2.0:1786177088.718296:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718303:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177088.718306:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177088.718308:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177088.718311:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235574016, offset 0 00000400:00000200:2.0:1786177088.718315:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177088.718319:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177088.718323:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177088.718326:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177088.718329:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177088.718332:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177088.718335:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177088.718338:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426551:426551:256:4294967295] 192.168.202.132@tcp LPNI seq info [426513:426513:8:4294967295] 00000400:00000200:2.0:1786177088.718343:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177088.718350:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177088.718355:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177088.718376:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177088.718395:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177088.718398:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786177088.718402:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177088.718405:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1786177088.718409:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177088.718411:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177088.718414:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177088.718418:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177088.718422:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177088.718424:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177088.718426:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177088.718430:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718440:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177088.718443:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718451:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.718452:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177088.718455:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177088.718459:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177088.718462:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177088.718464:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306435589 00000100:00000001:2.0:1786177088.718466:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306439424 00000100:00000001:2.0:1786177088.718469:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306435589 00000100:00000001:2.0:1786177088.718471:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306439424 00000100:00000001:2.0:1786177088.718474:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177088.718476:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.718478:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177088.718483:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177088/real 1786177088] req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177088.718492:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177088.718494:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177088.718496:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177088.718498:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177088.718500:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177088.718503:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177088.718506:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177088.718515:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718523:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177088.718524:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718526:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177088.718528:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177088.718531:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718538:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177088.718543:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718550:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177088.718553:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718554:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177088.718556:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718557:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177088.718559:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177088.718562:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177088.718565:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177088.718567:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177088.718571:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177088.718587:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574016:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177088.718591:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177088.718594:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718602:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177088.718603:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.718606:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177088.718608:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177088.718610:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177088.718613:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574144:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177088.718616:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177088.718618:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177088.718620:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.718622:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177088.718625:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177088.718627:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177088.718631:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177088.718634:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177088.718637:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177088.718640:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235574144, portal 4 00000100:00000001:2.0:1786177088.718642:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177088.718644:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487986560 : -110371221565056 : ffff9b9e328b9180) 00000100:00000040:2.0:1786177088.718650:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718658:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177088.718661:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177088.718663:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177088.718666:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235574144, offset 0 00000400:00000200:2.0:1786177088.718670:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177088.718674:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177088.718678:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177088.718681:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177088.718684:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177088.718687:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177088.718690:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177088.718693:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426552:426552:256:4294967295] 192.168.202.132@tcp LPNI seq info [426514:426514:8:4294967295] 00000400:00000200:2.0:1786177088.718698:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177088.718705:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177088.718709:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177088.718712:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177088.718716:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177088.718720:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786177088.718724:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177088.718728:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1786177088.718732:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177088.718734:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177088.718737:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177088.718741:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177088.718744:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177088.718745:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177088.718748:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177088.718752:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718761:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177088.718764:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718772:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.718773:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177088.718776:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177088.718779:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177088.718783:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177088.718785:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306435589 00000100:00000001:2.0:1786177088.718788:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306439424 00000100:00000001:2.0:1786177088.718790:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306435589 00000100:00000001:2.0:1786177088.718793:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306439424 00000100:00000001:2.0:1786177088.718795:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177088.718797:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.718799:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177088.718803:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177088/real 1786177088] req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177088.718811:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177088.718813:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177088.718814:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177088.718817:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177088.718819:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177088.718821:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177088.718824:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177088.718834:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718841:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177088.718843:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718845:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177088.718846:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177088.718850:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718858:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177088.718862:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718869:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177088.718872:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718874:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177088.718876:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718877:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177088.718879:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177088.718882:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177088.718885:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177088.718888:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177088.718892:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177088.718901:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574144:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177088.718905:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718907:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177088.718910:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9f80 x1872930235573888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718917:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177088.718919:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177088.718921:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:2.0:1786177088.718923:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177088.718925:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177088.718927:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177088.718929:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177088.718931:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:2.0:1786177088.718934:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177088.718935:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177088.718937:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9f80. 00000100:00000001:2.0:1786177088.718941:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177088.718942:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718945:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177088.718948:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb800 x1872930235574016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718962:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177088.718963:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177088.718965:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786177088.718967:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177088.718968:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177088.718970:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177088.718972:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177088.718973:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:2.0:1786177088.718975:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177088.718976:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177088.718978:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb800. 00000100:00000001:2.0:1786177088.718980:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177088.718981:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.718982:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177088.718986:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9180 x1872930235574144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.718993:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177088.718994:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177088.718996:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177088.718998:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177088.719000:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177088.719001:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177088.719003:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177088.719005:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:2.0:1786177088.719007:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177088.719008:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177088.719010:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9180. 00000100:00000001:2.0:1786177088.719012:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177088.719014:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.719016:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.719019:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177088.719020:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.719022:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177088.719024:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177088.719026:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177088.719030:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.719037:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177088.719039:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.719041:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177088.719043:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177088.719045:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177088.719048:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574272:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177088.719051:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177088.719053:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177088.719055:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177088.719057:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177088.719060:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177088.719062:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177088.719065:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177088.719068:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177088.719071:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177088.719074:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235574272, portal 4 00000100:00000001:2.0:1786177088.719076:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177088.719078:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487984768 : -110371221566848 : ffff9b9e328b8a80) 00000100:00000040:2.0:1786177088.719084:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.719091:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177088.719094:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177088.719096:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177088.719099:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235574272, offset 0 00000400:00000200:2.0:1786177088.719103:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177088.719106:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177088.719110:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177088.719113:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177088.719116:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177088.719118:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177088.719121:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177088.719125:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426553:426553:256:4294967295] 192.168.202.132@tcp LPNI seq info [426515:426515:8:4294967295] 00000400:00000200:2.0:1786177088.719129:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177088.719137:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177088.719142:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177088.719145:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177088.719149:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177088.719152:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786177088.719157:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177088.719160:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1786177088.719164:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177088.719166:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177088.719169:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177088.719173:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177088.719176:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177088.719178:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177088.719180:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177088.719184:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.719193:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177088.719196:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177113 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.719204:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.719206:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177088.719208:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177088.719211:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177088.719214:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177088.719216:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306435591 00000100:00000001:2.0:1786177088.719219:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306439424 00000100:00000001:2.0:1786177088.719221:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306435591 00000100:00000001:2.0:1786177088.719224:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306439424 00000100:00000001:2.0:1786177088.719226:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177088.719228:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.719230:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177088.719234:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177088/real 1786177088] req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177088.719243:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177088.719244:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177088.719246:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177088.719249:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177088.719251:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177088.719253:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177088.719256:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177088.719267:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.719274:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177088.719276:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.719278:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177088.719279:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177088.719283:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.719291:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177088.719295:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.719302:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177088.719305:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.719306:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177088.719307:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.719309:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177088.719311:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177088.719314:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177088.719317:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177088.719320:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177088.719324:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177088.719333:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574272:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177088.719337:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.719339:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177088.719342:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8a80 x1872930235574272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177113 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177088.719348:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177088.719350:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177088.719352:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177088.719354:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177088.719374:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177088.719376:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177088.719394:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177088.719396:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:2.0:1786177088.719400:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177088.719401:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177088.719402:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8a80. 00000100:00000001:2.0:1786177088.719405:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177088.719406:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.719408:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177088.719412:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177088.719413:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177088.719415:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177088.719417:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177089.229715:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177089.229724:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177089.229726:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177089.229732:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177089.229736:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177089.229740:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177089.229743:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177089.229746:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177089.229749:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177089.612475:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177089.612479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177089.612481:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177089.612484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177089.612485:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177089.612491:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177089.612491:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177089.612496:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177089.612497:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177089.612501:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177089.612501:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177089.612506:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177089.612506:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177089.612509:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177089.612511:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177089.612513:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177089.612513:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177089.612518:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177089.612520:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177089.612523:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177089.612524:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177089.612578:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177089.612583:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177089.612586:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177090.253352:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177090.253386:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177090.253389:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177090.253395:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177090.253398:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177090.253402:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177090.253405:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177090.253409:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177090.253413:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177090.636473:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177090.636477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177090.636480:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177090.636483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177090.636484:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177090.636490:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177090.636492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177090.636494:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177090.636495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177090.636499:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177090.636500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177090.636502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177090.636504:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177090.636508:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177090.636510:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177090.636580:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177090.636585:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177090.636589:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177091.277418:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177091.277426:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177091.277501:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177091.277508:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177091.277512:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177091.277516:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177091.277519:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177091.277522:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177091.277526:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177091.660469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177091.660472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177091.660477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177091.660479:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177091.660482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177091.660483:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177091.660485:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177091.660490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177091.660491:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177091.660492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177091.660498:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177091.660501:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177091.660504:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177091.660506:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177091.660508:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177091.660563:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177091.660568:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177091.660571:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177092.300515:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177092.300523:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177092.300525:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177092.300530:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177092.300535:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177092.300538:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177092.300540:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177092.300542:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177092.300544:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177092.620461:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306439424, at 4306439488 00000100:00000040:1.0:1786177092.620461:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306439424, at 4306439488 00000100:00000010:0.0:1786177092.620477:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c960. 00000100:00000010:1.0:1786177092.620479:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c420. 00000100:00000001:1.0:1786177092.684466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177092.684472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177092.684473:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177092.684476:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177092.684480:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:1.0:1786177092.684482:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177092.684484:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177092.684487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177092.684489:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177092.684492:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177092.684495:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177092.684495:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177092.684499:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177092.684500:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177092.684504:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177092.684505:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177092.684508:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177092.684514:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177092.684516:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177092.684520:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177092.684522:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177092.684524:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177092.684526:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177092.684529:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177092.684530:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177092.684532:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177092.684534:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177092.684589:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177092.684593:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177092.684595:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177093.324461:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177093.324468:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177093.324470:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177093.324475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177093.324477:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177093.324480:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.324482:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177093.324484:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.324486:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.708465:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177093.708469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177093.708472:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177093.708474:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177093.708477:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177093.708480:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177093.708482:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.708484:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177093.708487:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177093.708488:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177093.708491:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.708492:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177093.708494:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177093.708497:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.708497:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177093.708559:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177093.708564:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177093.708568:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177093.836454:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177093.836469:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177093.836471:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177093.836475:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177093.836478:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177093.836484:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11768 00000400:00000200:1.0:1786177093.836493:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177093.836498:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177093.836501:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177093.836503:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836506:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177093.836508:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836509:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177093.836513:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177093.836515:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177093.836517:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177093.836520:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177093.836522:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177093.836524:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177093.836526:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836528:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177093.836529:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836530:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177093.836533:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177093.836536:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836539:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177093.836541:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177093.836544:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177093.836547:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.836549:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177093.836552:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177093.836563:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328baa00. 00000020:00000040:1.0:1786177093.836567:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177093.836571:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177093.836573:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.836576:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177093.836580:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177093.836584:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177093.836591:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261acc00. 00000100:00000001:1.0:1786177093.836596:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177093.836606:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177093.836618:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177093.836623:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177093.836629:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ebce398] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177093.836662:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177093.836665:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177093.836667:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177093.836668:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177093.836670:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177093.836672:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177093.836674:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11768 00000400:00000200:1.0:1786177093.836677:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177093.836680:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177093.836682:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177093.836684:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836686:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177093.836688:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836689:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177093.836691:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177093.836692:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177093.836694:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177093.836696:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177093.836697:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177093.836698:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177093.836700:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1786177093.836701:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836706:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177093.836707:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177093.836707:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177093.836710:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177093.836712:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177093.836716:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000040:2.0:1786177093.836718:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177093.836723:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836729:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.836734:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1786177093.836735:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177093.836737:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177093.836742:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177093.836744:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177093.836744:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177093.836751:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177093.836751:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.836755:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177093.836760:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1786177093.836764:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235574400:192.168.202.132@tcp:250:kworker.0 00000008:00000001:1.0:1786177093.836767:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.836769:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1786177093.836770:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177093.836774:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:1.0:1786177093.836775:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bad80. 02000000:00000001:2.0:1786177093.836777:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177093.836779:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177093.836781:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1786177093.836786:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177093.836788:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.836790:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:2.0:1786177093.836790:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab400. 02000000:00000001:1.0:1786177093.836793:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177093.836795:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177093.836796:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177093.836802:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:1.0:1786177093.836803:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad800. 00000400:00000010:2.0:1786177093.836809:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00000001:1.0:1786177093.836810:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177093.836819:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00100000:1.0:1786177093.836820:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177093.836829:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1786177093.836829:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235574400, portal 25 00000100:00000001:1.0:1786177093.836833:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.836834:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177093.836838:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487992832 : -110371221558784 : ffff9b9e328baa00) 00000100:00000040:1.0:1786177093.836839:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b748f214] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177093.836848:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177093.836850:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177093.836855:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177093.836862:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177093.836862:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1786177093.836866:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:2.0:1786177093.836867:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00080000:1.0:1786177093.836869:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1786177093.836871:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000001:1.0:1786177093.836874:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:2.0:1786177093.836876:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235574400, offset 0 00000100:00080000:1.0:1786177093.836880:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11768 00000400:00000200:2.0:1786177093.836884:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177093.836885:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177093.836890:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177093.836892:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177093.836896:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177093.836897:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836901:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177093.836903:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177093.836906:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:1.0:1786177093.836907:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177093.836912:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177093.836913:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000400:00000200:2.0:1786177093.836915:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:1.0:1786177093.836916:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177093.836919:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:2.0:1786177093.836921:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000001:1.0:1786177093.836922:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786177093.836926:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177093.836928:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177093.836932:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426554:426554:256:4294967295] 192.168.202.132@tcp LPNI seq info [426516:426516:8:4294967295] 00000020:00000001:1.0:1786177093.836933:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786177093.836938:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177093.836940:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836943:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177093.836946:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836951:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177093.836955:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000020:00000040:1.0:1786177093.836956:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:1.0:1786177093.836962:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177093.836965:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177093.836969:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 02000000:00000001:1.0:1786177093.836970:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177093.836973:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177093.836977:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1786177093.836977:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00000001:1.0:1786177093.836982:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177093.836984:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177093.836987:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177093.836988:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177093.836989:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177093.836993:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00000010:1.0:1786177093.836995:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bbb80. 00000020:00000040:1.0:1786177093.837000:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000800:00000200:2.1:1786177093.837004:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:1.0:1786177093.837005:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177093.837008:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786177093.837010:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00000001:1.0:1786177093.837011:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177093.837014:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177093.837016:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1786177093.837017:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177093.837020:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177093.837024:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:1.0:1786177093.837025:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af800. 00000400:00000200:2.0:1786177093.837031:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1786177093.837032:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177093.837037:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000100:00100000:1.0:1786177093.837039:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1786177093.837041:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177093.837047:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1786177093.837050:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177093.837054:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177093.837055:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177093.837060:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3691ff3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837071:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1786177093.837073:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177093.837077:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1786177093.837078:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177093.837088:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177093.837091:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1786177093.837092:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177093.837093:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177093.837095:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177093.837097:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177093.837102:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11768 00000100:00000001:2.0:1786177093.837102:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177093.837112:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177093.837113:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177093.837117:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306444544 00000400:00000200:1.0:1786177093.837119:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177093.837122:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000010:2.0:1786177093.837123:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cc60. 00000100:00000040:1.0:1786177093.837125:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.837128:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177093.837130:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.837132:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177093.837133:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000020:00000001:1.0:1786177093.837136:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177093.837139:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177093.837140:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177093.837142:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:2.0:1786177093.837143:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306444544 00000020:00000001:1.0:1786177093.837144:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000010:2.0:1786177093.837147:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c8a0. 00000020:00000001:1.0:1786177093.837148:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:2.0:1786177093.837151:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000040:1.0:1786177093.837156:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177093.837156:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177093.837161:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.837163:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177093.837165:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1786177093.837169:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00100000:2.0:1786177093.837171:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177093/real 1786177093] req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177093.837174:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177093.837181:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:1.0:1786177093.837190:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000001:2.0:1786177093.837195:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1786177093.837196:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1786177093.837197:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1786177093.837200:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1786177093.837201:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 02000000:00000001:1.0:1786177093.837206:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1786177093.837206:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177093.837210:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 02000000:00000001:1.0:1786177093.837211:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.837213:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1786177093.837215:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177093.837219:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000200:2.0:1786177093.837219:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1786177093.837227:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.837233:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177093.837236:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1786177093.837241:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8000. 00000100:00000001:2.0:1786177093.837242:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177093.837245:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177093.837247:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:2.0:1786177093.837248:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177093.837251:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177093.837254:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1786177093.837255:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177093.837258:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.837264:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177093.837266:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177093.837270:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.837270:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1786177093.837278:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad400. 00000100:00000040:2.0:1786177093.837280:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177093.837283:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.837289:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:1.0:1786177093.837291:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177093.837293:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.837297:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177093.837300:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.837305:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177093.837306:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177093.837308:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177093.837311:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177093.837312:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:1.0:1786177093.837317:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ab5f277] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177093.837319:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177093.837324:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1786177093.837344:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177093.837345:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177093.837349:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177093.837383:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1786177093.837384:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235574400:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177093.837388:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1786177093.837391:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1786177093.837393:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177093.837394:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177093.837400:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328baa00 x1872930235574400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177093.837401:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177093.837408:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11768 00000100:00000001:2.0:1786177093.837409:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177093.837412:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177093.837415:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab400. 00000400:00000200:1.0:1786177093.837418:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177093.837420:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177093.837424:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177093.837426:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000400:00000200:1.0:1786177093.837428:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000020:00000001:2.0:1786177093.837429:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00080000:1.0:1786177093.837431:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 02000000:00000010:2.0:1786177093.837432:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261acc00. 00000100:00000040:1.0:1786177093.837437:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:2.0:1786177093.837438:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177093.837442:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1786177093.837443:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:2.0:1786177093.837444:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328baa00. 00000100:00000040:1.0:1786177093.837447:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.837450:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177093.837450:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:1.0:1786177093.837454:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177093.837454:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177093.837457:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000001:2.0:1786177093.837459:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1786177093.837460:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177093.837463:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:1.0:1786177093.837464:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:1.0:1786177093.837467:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177093.837467:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177093.837471:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177093.837473:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177093.837475:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177093.837475:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1786177093.837479:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177093.837479:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177093.837482:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177093.837483:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177093.837486:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000040:2.0:1786177093.837489:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177093.837492:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177093.837497:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177093.837500:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177093.837503:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177093.837504:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.837506:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177093.837507:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177093.837511:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.837514:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177093.837515:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.837518:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177093.837520:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9500. 02000000:00000001:2.0:1786177093.837521:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177093.837525:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1786177093.837526:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177093.837527:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177093.837529:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1786177093.837533:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574528:192.168.202.132@tcp:38:kworker.0 02000000:00000001:1.0:1786177093.837534:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177093.837540:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1786177093.837541:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177093.837544:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177093.837545:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177093.837546:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177093.837549:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1786177093.837550:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177093.837552:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786177093.837555:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177093.837559:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae000. 00000400:00000200:2.0:1786177093.837559:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177093.837564:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00000001:1.0:1786177093.837566:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177093.837573:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00100000:1.0:1786177093.837576:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1786177093.837577:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235574528, portal 10 00000100:00000001:2.0:1786177093.837581:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177093.837584:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487993728 : -110371221557888 : ffff9b9e328bad80) 00000100:00000001:1.0:1786177093.837592:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1786177093.837595:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177093.837596:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177093.837606:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000cd924b3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837606:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177093.837613:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177093.837616:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177093.837619:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235574528, offset 0 00000400:00000200:2.0:1786177093.837628:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177093.837631:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177093.837632:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177093.837639:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000040:1.0:1786177093.837640:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11778) 00000400:00000200:2.0:1786177093.837642:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177093.837646:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177093.837649:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177093.837651:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177093.837653:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426555:426555:256:4294967295] 192.168.202.132@tcp LPNI seq info [426517:426517:8:4294967295] 00000400:00000200:2.0:1786177093.837657:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177093.837663:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177093.837667:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177093.837669:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177093.837672:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177093.837674:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786177093.837677:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177093.837679:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1786177093.837681:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177093.837683:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177093.837685:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177093.837687:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177093.837689:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177093.837691:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177093.837692:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177093.837695:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837700:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177093.837702:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837706:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.837707:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177093.837709:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177093.837712:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177093.837714:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177093.837715:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306440708 00000100:00000001:2.0:1786177093.837717:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306444544 00000100:00000001:2.0:1786177093.837718:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306440708 00000100:00000001:2.0:1786177093.837720:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306444544 00000100:00000001:2.0:1786177093.837722:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177093.837723:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.837724:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177093.837726:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177093/real 1786177093] req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177093.837731:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177093.837732:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177093.837733:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177093.837735:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177093.837736:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177093.837737:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177093.837739:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177093.837744:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837748:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177093.837749:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.837750:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177093.837751:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177093.837753:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837757:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177093.837759:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837763:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177093.837765:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.837766:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177093.837766:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.837768:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177093.837769:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177093.837771:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177093.837774:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177093.837775:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177093.837777:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177093.837782:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574528:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177093.837785:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177093.837787:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837791:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177093.837792:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177093.837793:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177093.837794:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177093.837796:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177093.837797:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574656:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177093.837799:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177093.837800:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177093.837801:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177093.837803:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177093.837805:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177093.837806:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177093.837808:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177093.837810:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177093.837812:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177093.837813:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235574656, portal 10 00000100:00000001:2.0:1786177093.837815:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177093.837816:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487997312 : -110371221554304 : ffff9b9e328bbb80) 00000100:00000040:2.0:1786177093.837819:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837823:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177093.837825:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177093.837826:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177093.837828:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235574656, offset 0 00000400:00000200:2.0:1786177093.837830:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177093.837832:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177093.837834:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177093.837836:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177093.837838:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177093.837840:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177093.837841:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177093.837843:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426556:426556:256:4294967295] 192.168.202.132@tcp LPNI seq info [426518:426518:8:4294967295] 00000400:00000200:2.0:1786177093.837846:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177093.837850:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177093.837852:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177093.837854:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177093.837856:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177093.837858:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786177093.837861:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177093.837862:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1786177093.837865:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177093.837867:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177093.837868:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177093.837871:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177093.837872:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177093.837874:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177093.837875:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177093.837877:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837882:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177093.837884:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837888:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.837889:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177093.837891:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177093.837893:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177093.837894:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177093.837896:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306440708 00000100:00000001:2.0:1786177093.837897:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306444544 00000100:00000001:2.0:1786177093.837899:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306440708 00000100:00000001:2.0:1786177093.837900:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306444544 00000100:00000001:2.0:1786177093.837902:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177093.837903:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.837904:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177093.837906:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177093/real 1786177093] req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177093.837911:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177093.837912:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177093.837912:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177093.837914:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177093.837915:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177093.837916:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177093.837918:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177093.837923:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837927:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177093.837928:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.837929:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177093.837930:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177093.837932:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837936:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177093.837938:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837942:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177093.837943:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.837944:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177093.837945:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.837947:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177093.837948:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177093.837950:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177093.837952:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177093.837953:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177093.837956:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177093.837960:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574656:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177093.837962:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177093.837964:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.837968:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177093.837969:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177093.837970:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177093.837971:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177093.837972:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177093.837974:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574784:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177093.837976:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177093.837977:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177093.837978:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177093.837979:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177093.837981:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa400. 02000000:00000001:2.0:1786177093.837982:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177093.837985:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177093.837986:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177093.837988:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177093.837990:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235574784, portal 4 00000100:00000001:2.0:1786177093.837991:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177093.837992:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982080 : -110371221569536 : ffff9b9e328b8000) 00000100:00000040:2.0:1786177093.837996:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838001:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177093.838002:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177093.838004:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177093.838005:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235574784, offset 0 00000400:00000200:2.0:1786177093.838007:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177093.838009:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177093.838011:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177093.838012:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177093.838014:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177093.838015:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177093.838017:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177093.838018:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426557:426557:256:4294967295] 192.168.202.132@tcp LPNI seq info [426519:426519:8:4294967295] 00000400:00000200:2.0:1786177093.838021:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177093.838025:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177093.838028:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177093.838029:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177093.838032:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177093.838033:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786177093.838035:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177093.838038:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1786177093.838041:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177093.838042:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177093.838044:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177093.838046:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177093.838048:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177093.838049:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177093.838050:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177093.838052:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838057:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177093.838059:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838067:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.838068:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177093.838070:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177093.838072:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177093.838074:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177093.838075:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306440708 00000100:00000001:2.0:1786177093.838077:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306444544 00000100:00000001:2.0:1786177093.838078:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306440708 00000100:00000001:2.0:1786177093.838079:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306444544 00000100:00000001:2.0:1786177093.838081:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177093.838082:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.838083:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177093.838085:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177093/real 1786177093] req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177093.838089:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177093.838090:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177093.838091:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177093.838092:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177093.838093:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177093.838094:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177093.838096:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177093.838102:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838106:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177093.838106:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838107:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177093.838108:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177093.838110:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838115:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177093.838117:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838121:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177093.838123:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838124:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177093.838125:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838126:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177093.838126:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177093.838128:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177093.838130:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177093.838131:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177093.838133:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177093.838138:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574784:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177093.838140:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838141:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177093.838143:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bad80 x1872930235574528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838147:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177093.838148:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177093.838149:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786177093.838150:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177093.838151:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177093.838152:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177093.838153:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177093.838154:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:2.0:1786177093.838156:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177093.838157:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177093.838158:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bad80. 00000100:00000001:2.0:1786177093.838161:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177093.838162:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838163:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177093.838165:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bbb80 x1872930235574656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838169:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177093.838170:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177093.838171:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177093.838172:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177093.838173:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177093.838173:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177093.838174:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177093.838175:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af800. 02000000:00000001:2.0:1786177093.838176:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177093.838177:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177093.838178:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bbb80. 00000100:00000001:2.0:1786177093.838179:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177093.838180:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838181:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177093.838182:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8000 x1872930235574784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838186:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177093.838187:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177093.838188:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:2.0:1786177093.838189:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177093.838189:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177093.838190:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177093.838191:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177093.838192:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:2.0:1786177093.838193:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177093.838194:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177093.838195:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8000. 00000100:00000001:2.0:1786177093.838196:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177093.838197:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838198:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838200:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177093.838201:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.838203:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177093.838204:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177093.838204:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177093.838207:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838211:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177093.838212:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177093.838213:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177093.838215:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177093.838216:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177093.838218:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574912:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177093.838220:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177093.838221:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177093.838221:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177093.838223:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177093.838224:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa400. 02000000:00000001:2.0:1786177093.838225:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177093.838227:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177093.838228:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177093.838230:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177093.838232:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235574912, portal 4 00000100:00000001:2.0:1786177093.838233:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177093.838234:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487987456 : -110371221564160 : ffff9b9e328b9500) 00000100:00000040:2.0:1786177093.838238:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838242:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177093.838244:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177093.838245:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177093.838247:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235574912, offset 0 00000400:00000200:2.0:1786177093.838249:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177093.838252:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177093.838254:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177093.838256:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177093.838257:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177093.838259:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177093.838261:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177093.838263:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426558:426558:256:4294967295] 192.168.202.132@tcp LPNI seq info [426520:426520:8:4294967295] 00000400:00000200:2.0:1786177093.838265:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177093.838269:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177093.838272:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177093.838273:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177093.838276:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177093.838278:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786177093.838280:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177093.838282:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1786177093.838285:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177093.838286:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177093.838288:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177093.838290:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177093.838292:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177093.838293:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177093.838295:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177093.838297:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838302:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177093.838304:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177118 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838308:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.838309:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177093.838310:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177093.838312:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177093.838313:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177093.838315:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306440710 00000100:00000001:2.0:1786177093.838317:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306444544 00000100:00000001:2.0:1786177093.838318:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306440710 00000100:00000001:2.0:1786177093.838320:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306444544 00000100:00000001:2.0:1786177093.838321:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177093.838322:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.838323:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177093.838325:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177093/real 1786177093] req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177093.838341:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177093.838342:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177093.838343:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177093.838345:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177093.838346:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177093.838348:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177093.838350:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177093.838370:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838375:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177093.838376:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838378:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177093.838379:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177093.838381:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838386:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177093.838388:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838392:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177093.838394:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838396:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177093.838396:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838398:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177093.838399:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177093.838400:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177093.838402:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177093.838403:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177093.838406:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177093.838410:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235574912:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177093.838413:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838414:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177093.838416:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9500 x1872930235574912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177118 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177093.838420:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177093.838421:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177093.838422:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:2.0:1786177093.838424:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177093.838425:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177093.838425:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177093.838427:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177093.838428:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:2.0:1786177093.838430:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177093.838431:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177093.838432:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9500. 00000100:00000001:2.0:1786177093.838433:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177093.838434:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838435:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177093.838437:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177093.838438:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177093.838439:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177093.838441:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177094.348462:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177094.348471:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177094.348474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177094.348480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177094.348485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177094.348489:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177094.348491:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177094.348494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177094.348498:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177094.732464:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177094.732470:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177094.732474:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177094.732476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177094.732479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177094.732481:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177094.732482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177094.732486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177094.732487:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177094.732493:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177094.732497:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177094.732502:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177094.732506:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177094.732508:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177094.732512:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177094.732542:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177094.732545:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177094.732548:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177095.372452:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177095.372461:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177095.372463:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177095.372469:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177095.372473:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177095.372477:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177095.372480:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177095.372483:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177095.372486:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177095.756469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177095.756474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177095.756478:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177095.756479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177095.756482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177095.756488:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177095.756489:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177095.756492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177095.756495:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177095.756498:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177095.756500:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177095.756502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177095.756506:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177095.756510:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177095.756514:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177095.756569:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177095.756572:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177095.756575:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177096.396483:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177096.396493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177096.396496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177096.396502:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177096.396506:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177096.396510:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177096.396513:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177096.396516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177096.396519:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177096.780468:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177096.780474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177096.780477:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177096.780479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177096.780482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177096.780487:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177096.780488:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177096.780491:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177096.780494:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177096.780497:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177096.780500:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177096.780501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177096.780506:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177096.780510:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177096.780514:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177096.780568:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177096.780572:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177096.780575:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177097.420467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177097.420475:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177097.420476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177097.420481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177097.420485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177097.420488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177097.420490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177097.420493:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177097.420495:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177097.740467:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306444544, at 4306444608 00000100:00000040:1.0:1786177097.740471:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306444544, at 4306444608 00000100:00000010:0.0:1786177097.740482:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cc60. 00000100:00000010:1.0:1786177097.740484:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c8a0. 00000100:00000001:1.0:1786177097.804466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177097.804468:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177097.804472:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177097.804475:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177097.804477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177097.804479:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177097.804480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177097.804485:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177097.804486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177097.804488:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177097.804490:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177097.804494:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177097.804496:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177097.804498:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177097.804500:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177097.804546:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177097.804549:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177097.804552:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177098.444596:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177098.444604:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177098.444606:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177098.444612:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177098.444618:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177098.444624:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177098.444627:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177098.444630:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177098.444634:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177098.828437:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177098.828437:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177098.828446:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.828447:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177098.828451:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177098.828454:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177098.828456:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177098.828457:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177098.828459:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177098.828461:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177098.828462:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.828468:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.828472:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177098.828474:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.828477:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177098.828539:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177098.828545:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177098.828551:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177098.956515:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177098.956527:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177098.956530:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177098.956534:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177098.956539:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177098.956547:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11773 00000400:00000200:2.0:1786177098.956565:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177098.956572:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177098.956575:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177098.956578:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956582:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177098.956584:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956586:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177098.956590:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177098.956592:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177098.956594:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177098.956597:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177098.956599:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177098.956601:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177098.956604:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956605:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177098.956607:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956608:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177098.956611:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177098.956613:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956615:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177098.956617:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177098.956621:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177098.956624:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177098.956626:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177098.956628:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177098.956640:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 00000020:00000040:2.0:1786177098.956643:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177098.956648:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177098.956649:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177098.956652:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177098.956655:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177098.956660:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177098.956669:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:2.0:1786177098.956675:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177098.956685:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177098.956694:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177098.956699:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177098.956703:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177098.956725:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177098.956728:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177098.956731:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177098.956732:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177098.956734:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177098.956736:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177098.956738:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11773 00000400:00000200:2.0:1786177098.956740:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177098.956742:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177098.956744:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177098.956747:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956749:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177098.956752:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956753:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177098.956756:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177098.956757:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177098.956758:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177098.956760:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177098.956761:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177098.956762:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177098.956764:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956766:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177098.956767:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956768:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177098.956770:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1786177098.956770:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177098.956775:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177098.956777:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177098.956778:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.956781:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1786177098.956781:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177098.956787:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177098.956790:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177098.956792:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177098.956793:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177098.956799:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177098.956803:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177098.956807:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.956808:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177098.956811:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177098.956811:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786177098.956817:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177098.956819:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177098.956820:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177098.956823:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177098.956824:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177098.956825:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177098.956829:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177098.956829:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177098.956833:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177098.956837:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235575040:192.168.202.132@tcp:250:kworker.0 02000000:00000010:2.0:1786177098.956839:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786177098.956844:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.956848:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1786177098.956850:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177098.956853:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177098.956858:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177098.956859:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1786177098.956861:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1786177098.956864:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177098.956868:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177098.956870:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177098.956876:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177098.956883:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177098.956885:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177098.956886:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:2.0:1786177098.956892:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:1.0:1786177098.956893:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00000001:2.0:1786177098.956894:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177098.956897:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177098.956901:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:1.0:1786177098.956904:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00080000:2.0:1786177098.956904:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11773 00000400:00000200:2.0:1786177098.956911:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177098.956913:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177098.956914:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177098.956916:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956918:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177098.956919:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:1.0:1786177098.956921:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235575040, portal 25 00000100:00000001:2.0:1786177098.956921:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177098.956927:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1786177098.956928:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177098.956931:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000100:00000001:1.0:1786177098.956932:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469630848 : -110371239920768 : ffff9b9e31737b80) 00000020:00000001:2.0:1786177098.956933:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177098.956936:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177098.956939:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177098.956942:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177098.956943:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177098.956947:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956951:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177098.956956:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956960:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177098.956967:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:2.0:1786177098.956967:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177098.956971:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000040:1.0:1786177098.956976:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177098.956977:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177098.956982:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 02000000:00000001:2.0:1786177098.956982:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177098.956986:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1786177098.956987:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235575040, offset 0 02000000:00000001:2.0:1786177098.956989:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177098.956992:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177098.956994:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177098.956998:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177098.956999:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177098.957001:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177098.957006:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786177098.957008:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177098.957011:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1786177098.957012:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177098.957013:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177098.957017:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177098.957018:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177098.957019:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177098.957023:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab400. 00000400:00000200:1.0:1786177098.957026:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:2.0:1786177098.957026:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177098.957036:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:2.0:1786177098.957037:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177098.957043:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177098.957050:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177098.957055:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.957060:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177098.957061:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426559:426559:256:4294967295] 192.168.202.132@tcp LPNI seq info [426521:426521:8:4294967295] 00000100:00000001:2.0:1786177098.957067:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177098.957071:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177098.957078:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177098.957086:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177098.957100:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177098.957100:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177098.957105:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1786177098.957108:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00080000:2.0:1786177098.957109:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177098.957113:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177098.957115:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177098.957118:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177098.957123:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11773 00000800:00000200:1.0:1786177098.957125:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:2.0:1786177098.957130:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:1.0:1786177098.957132:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000400:00000200:2.0:1786177098.957137:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177098.957140:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1786177098.957143:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:2.0:1786177098.957146:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:1.1:1786177098.957149:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1786177098.957150:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177098.957155:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.957160:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177098.957162:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177098.957163:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000800:00000100:1.0:1786177098.957164:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000020:00000001:2.0:1786177098.957165:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000010:1.0:1786177098.957170:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000020:00000040:2.0:1786177098.957170:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177098.957176:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177098.957178:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177098.957181:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177098.957182:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1786177098.957185:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177098.957192:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1786177098.957192:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.957198:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177098.957199:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177098.957202:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000020:00000040:2.0:1786177098.957205:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:1.0:1786177098.957208:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1786177098.957212:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000200:1.0:1786177098.957216:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177098.957219:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177098.957225:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177098.957231:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177098.957233:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1786177098.957236:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177098.957239:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177098.957240:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177098.957245:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177098.957250:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957253:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177098.957253:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957258:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:2.0:1786177098.957260:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000020:00000040:2.0:1786177098.957263:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177098.957265:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177098.957266:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177098.957270:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177098.957271:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177098.957275:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177098.957276:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306449664 02000000:00000001:2.0:1786177098.957278:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177098.957296:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177098.957301:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:2.0:1786177098.957304:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177098.957309:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:1.0:1786177098.957310:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c626c0. 00000100:00000001:2.0:1786177098.957319:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177098.957321:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177098.957321:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177098.957328:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957332:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306449664 00000100:00000001:2.0:1786177098.957334:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177098.957338:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c625a0. 00000100:00080000:2.0:1786177098.957339:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177098.957344:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177098.957346:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177098.957346:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177098.957350:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177098.957352:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177098.957354:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177098.957382:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957386:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1786177098.957387:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11773 00000100:00100000:1.0:1786177098.957393:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177098/real 1786177098] req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177098.957394:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177098.957408:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177098.957412:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000001:1.0:1786177098.957414:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1786177098.957416:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000001:1.0:1786177098.957418:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1786177098.957421:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1786177098.957422:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000100:00000040:2.0:1786177098.957426:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177098.957427:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000100:00000001:2.0:1786177098.957429:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:1.0:1786177098.957431:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000020:00000001:2.0:1786177098.957432:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177098.957436:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:2.0:1786177098.957436:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177098.957438:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177098.957443:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000200:1.0:1786177098.957444:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177098.957447:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177098.957453:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177098.957460:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000200:1.0:1786177098.957462:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177098.957463:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177098.957469:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177098.957474:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177098.957479:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:1.0:1786177098.957480:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177098.957484:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1786177098.957484:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177098.957489:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957490:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1786177098.957492:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177098.957493:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1786177098.957496:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1786177098.957501:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177098.957501:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177098.957506:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177098.957508:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177098.957513:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957516:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177098.957517:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177098.957522:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734700. 00000100:00000040:1.0:1786177098.957524:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177098.957526:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177098.957531:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177098.957533:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177098.957536:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177098.957539:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177098.957541:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177098.957546:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:1.0:1786177098.957549:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177098.957550:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957556:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177098.957559:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177098.957561:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177098.957565:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.957572:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177098.957572:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177098.957575:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957577:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1786177098.957581:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088313762] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177098.957583:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177098.957593:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177098.957593:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957596:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177098.957600:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11783) 00000100:00000040:1.0:1786177098.957603:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177098.957612:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235575040:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177098.957616:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.957619:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177098.957623:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737b80 x1872930235575040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957628:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177098.957630:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177098.957633:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177098.957635:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177098.957637:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177098.957638:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177098.957640:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177098.957642:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:1.0:1786177098.957645:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177098.957647:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177098.957648:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737b80. 00000100:00000001:1.0:1786177098.957651:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177098.957652:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.957654:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.957661:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177098.957662:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957664:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177098.957665:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177098.957666:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177098.957669:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957674:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177098.957675:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177098.957677:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177098.957678:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177098.957680:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177098.957682:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575168:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177098.957685:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177098.957686:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177098.957687:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177098.957689:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177098.957691:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177098.957693:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177098.957696:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177098.957698:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177098.957701:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177098.957704:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235575168, portal 10 00000100:00000001:1.0:1786177098.957705:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177098.957706:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620992 : -110371239930624 : ffff9b9e31735500) 00000100:00000040:1.0:1786177098.957712:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957716:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177098.957718:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177098.957720:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177098.957721:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235575168, offset 0 00000400:00000200:1.0:1786177098.957724:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177098.957727:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177098.957730:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177098.957732:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177098.957734:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177098.957736:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177098.957738:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177098.957741:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426560:426560:256:4294967295] 192.168.202.132@tcp LPNI seq info [426522:426522:8:4294967295] 00000400:00000200:1.0:1786177098.957744:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177098.957749:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177098.957752:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177098.957754:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177098.957757:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177098.957760:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786177098.957763:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177098.957765:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1786177098.957767:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177098.957769:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177098.957771:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177098.957773:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177098.957776:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177098.957777:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177098.957779:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177098.957781:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957787:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177098.957789:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957793:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957794:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177098.957796:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177098.957799:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177098.957801:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177098.957802:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306445828 00000100:00000001:1.0:1786177098.957805:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306449664 00000100:00000001:1.0:1786177098.957806:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306445828 00000100:00000001:1.0:1786177098.957808:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306449664 00000100:00000001:1.0:1786177098.957810:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177098.957811:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957812:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177098.957815:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177098/real 1786177098] req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177098.957819:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177098.957820:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177098.957822:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177098.957823:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177098.957825:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177098.957826:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177098.957828:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177098.957833:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957837:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177098.957838:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.957839:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177098.957840:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177098.957842:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957846:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177098.957848:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957853:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177098.957855:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.957856:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177098.957857:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.957858:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177098.957860:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177098.957862:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177098.957863:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177098.957865:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177098.957867:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177098.957872:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575168:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177098.957875:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177098.957877:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957882:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177098.957883:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177098.957884:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177098.957885:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177098.957886:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177098.957888:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575296:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177098.957890:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177098.957891:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177098.957892:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177098.957894:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177098.957896:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af000. 02000000:00000001:1.0:1786177098.957898:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177098.957900:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177098.957901:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177098.957903:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177098.957905:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235575296, portal 10 00000100:00000001:1.0:1786177098.957906:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177098.957907:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469622784 : -110371239928832 : ffff9b9e31735c00) 00000100:00000040:1.0:1786177098.957912:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957917:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177098.957918:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177098.957920:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177098.957921:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235575296, offset 0 00000400:00000200:1.0:1786177098.957924:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177098.957926:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177098.957928:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177098.957929:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177098.957932:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177098.957933:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177098.957935:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177098.957937:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426561:426561:256:4294967295] 192.168.202.132@tcp LPNI seq info [426523:426523:8:4294967295] 00000400:00000200:1.0:1786177098.957940:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177098.957944:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177098.957946:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177098.957948:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177098.957951:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177098.957952:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786177098.957956:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177098.957957:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1786177098.957961:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177098.957963:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177098.957965:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177098.957967:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177098.957969:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177098.957971:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177098.957972:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177098.957975:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957980:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177098.957982:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.957986:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.957987:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177098.957989:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177098.957991:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177098.957994:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177098.957995:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306445828 00000100:00000001:1.0:1786177098.957997:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306449664 00000100:00000001:1.0:1786177098.957999:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306445828 00000100:00000001:1.0:1786177098.958001:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306449664 00000100:00000001:1.0:1786177098.958003:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177098.958004:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.958005:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177098.958008:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177098/real 1786177098] req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177098.958012:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177098.958013:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177098.958014:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177098.958015:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177098.958016:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177098.958017:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177098.958019:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177098.958025:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958029:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177098.958030:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958031:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177098.958032:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177098.958034:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958038:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177098.958040:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958044:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177098.958046:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958047:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177098.958047:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958049:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177098.958051:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177098.958053:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177098.958055:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177098.958057:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177098.958059:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177098.958064:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575296:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177098.958066:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177098.958068:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958072:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177098.958072:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177098.958074:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177098.958075:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177098.958076:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177098.958078:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575424:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177098.958079:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177098.958080:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177098.958081:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177098.958082:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177098.958085:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177098.958086:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177098.958089:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177098.958090:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177098.958092:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177098.958094:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235575424, portal 4 00000100:00000001:1.0:1786177098.958096:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177098.958097:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469621888 : -110371239929728 : ffff9b9e31735880) 00000100:00000040:1.0:1786177098.958101:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958106:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177098.958107:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177098.958109:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177098.958111:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235575424, offset 0 00000400:00000200:1.0:1786177098.958113:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177098.958115:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177098.958117:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177098.958118:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177098.958120:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177098.958121:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177098.958123:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177098.958125:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426562:426562:256:4294967295] 192.168.202.132@tcp LPNI seq info [426524:426524:8:4294967295] 00000400:00000200:1.0:1786177098.958128:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177098.958132:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177098.958135:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177098.958136:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177098.958138:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177098.958140:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786177098.958143:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177098.958145:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1786177098.958147:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177098.958149:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177098.958151:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177098.958153:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177098.958154:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177098.958155:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177098.958157:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177098.958159:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958164:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177098.958166:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958175:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.958176:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177098.958178:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177098.958180:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177098.958182:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177098.958183:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306445828 00000100:00000001:1.0:1786177098.958185:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306449664 00000100:00000001:1.0:1786177098.958187:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306445828 00000100:00000001:1.0:1786177098.958188:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306449664 00000100:00000001:1.0:1786177098.958189:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177098.958190:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.958191:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177098.958194:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177098/real 1786177098] req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177098.958198:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177098.958199:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177098.958200:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177098.958201:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177098.958202:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177098.958203:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177098.958206:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177098.958211:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958215:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177098.958216:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958217:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177098.958217:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177098.958220:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958225:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177098.958227:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958232:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177098.958233:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958234:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177098.958235:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958236:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177098.958237:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177098.958238:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177098.958240:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177098.958241:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177098.958243:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177098.958248:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575424:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177098.958250:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177098.958252:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958256:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177098.958257:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177098.958258:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177098.958259:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177098.958261:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177098.958262:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575552:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177098.958265:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177098.958266:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177098.958268:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177098.958269:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177098.958271:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177098.958272:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177098.958274:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177098.958276:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177098.958278:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000200:1.0:1786177098.958292:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235575552, portal 4 00000100:00000001:1.0:1786177098.958295:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177098.958296:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469617408 : -110371239934208 : ffff9b9e31734700) 00000100:00000040:1.0:1786177098.958301:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958306:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177098.958307:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177098.958309:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000200:1.0:1786177098.958311:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235575552, offset 0 00000400:00000200:1.0:1786177098.958314:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177098.958317:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177098.958319:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177098.958321:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177098.958323:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177098.958325:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177098.958326:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177098.958329:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426563:426563:256:4294967295] 192.168.202.132@tcp LPNI seq info [426525:426525:8:4294967295] 00000400:00000200:1.0:1786177098.958331:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177098.958335:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177098.958338:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177098.958340:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177098.958342:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177098.958344:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1786177098.958347:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177098.958349:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1786177098.958351:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177098.958353:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177098.958355:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177098.958372:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177098.958375:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:1.0:1786177098.958376:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:1.0:1786177098.958378:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177098.958380:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958386:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177098.958388:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177123 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958393:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.958394:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177098.958396:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177098.958398:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177098.958400:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177098.958402:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306445831 00000100:00000001:1.0:1786177098.958404:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306449664 00000100:00000001:1.0:1786177098.958405:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306445831 00000100:00000001:1.0:1786177098.958407:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306449664 00000100:00000001:1.0:1786177098.958408:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177098.958409:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.958410:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177098.958413:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177098/real 1786177098] req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177098.958418:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177098.958419:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177098.958420:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177098.958422:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177098.958423:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177098.958425:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177098.958427:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177098.958433:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958437:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177098.958437:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958439:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177098.958439:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177098.958442:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958445:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177098.958448:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958452:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177098.958453:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958454:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177098.958455:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958456:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177098.958457:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177098.958458:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177098.958461:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177098.958462:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177098.958465:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177098.958470:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575552:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177098.958473:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958474:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177098.958476:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735500 x1872930235575168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958481:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177098.958482:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177098.958484:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177098.958485:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177098.958486:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177098.958487:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177098.958488:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177098.958490:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:1.0:1786177098.958492:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177098.958493:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177098.958494:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735500. 00000100:00000001:1.0:1786177098.958496:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177098.958497:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958498:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177098.958500:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735c00 x1872930235575296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958505:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177098.958506:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177098.958506:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af000. 02000000:00000001:1.0:1786177098.958508:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177098.958508:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177098.958509:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177098.958510:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177098.958511:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:1.0:1786177098.958513:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177098.958513:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177098.958514:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735c00. 00000100:00000001:1.0:1786177098.958515:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177098.958516:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958517:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177098.958519:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735880 x1872930235575424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958524:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177098.958527:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177098.958528:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177098.958530:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177098.958532:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177098.958533:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177098.958535:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177098.958537:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:1.0:1786177098.958539:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177098.958540:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177098.958541:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735880. 00000100:00000001:1.0:1786177098.958544:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177098.958545:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958547:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177098.958551:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734700 x1872930235575552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177123 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177098.958558:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177098.958560:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177098.958562:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177098.958564:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177098.958566:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177098.958567:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177098.958570:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177098.958572:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:1.0:1786177098.958574:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177098.958575:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177098.958577:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734700. 00000100:00000001:1.0:1786177098.958579:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177098.958580:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958582:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177098.958584:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177098.958585:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177098.958587:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177098.958588:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177099.468737:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177099.468747:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177099.468750:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177099.468756:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177099.468760:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177099.468763:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177099.468765:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177099.468767:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177099.468770:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177099.852469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177099.852476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177099.852478:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177099.852481:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177099.852482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177099.852490:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177099.852491:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177099.852496:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177099.852496:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177099.852500:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177099.852502:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177099.852504:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177099.852505:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177099.852509:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177099.852510:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177099.852514:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177099.852515:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177099.852520:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177099.852525:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177099.852529:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177099.852531:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177099.852600:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177099.852605:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177099.852610:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177100.492450:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177100.492458:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177100.492462:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177100.492484:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177100.492489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177100.492495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177100.492498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177100.492505:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177100.492509:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177100.876446:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177100.876454:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177100.876461:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177100.876464:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177100.876469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177100.876471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177100.876504:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177100.876513:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177100.876515:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177100.876520:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177100.876526:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177100.876531:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177100.876533:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177100.876537:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177100.876540:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177100.876541:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177100.876547:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177100.876568:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177101.516502:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177101.516509:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177101.516512:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177101.516517:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177101.516520:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177101.516523:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177101.516526:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177101.516528:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177101.516532:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177101.900435:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177101.900442:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177101.900444:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177101.900445:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177101.900449:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177101.900450:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177101.900453:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177101.900454:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177101.900455:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177101.900457:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177101.900458:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177101.900459:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177101.900460:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177101.900463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177101.900466:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177101.900518:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177101.900521:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177101.900524:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177102.540438:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177102.540444:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177102.540446:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177102.540451:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177102.540454:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177102.540457:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177102.540460:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177102.540464:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177102.540466:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177102.860465:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306449664, at 4306449728 00000100:00000040:1.0:1786177102.860465:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306449664, at 4306449728 00000100:00000010:0.0:1786177102.860480:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c626c0. 00000100:00000010:1.0:1786177102.860483:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c625a0. 00010000:00000001:0.0:1786177102.924417:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177102.924422:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177102.924425:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177102.924429:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177102.924429:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177102.924434:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177102.924435:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177102.924441:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:1.0:1786177102.924443:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177102.924444:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177102.924446:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177102.924450:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177102.924451:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177102.924453:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177102.924456:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177102.924457:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177102.924459:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177102.924463:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177102.924464:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177102.924465:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177102.924468:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177102.924470:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177102.924470:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177102.924473:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177102.924476:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177102.924476:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177102.924487:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177102.924533:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177102.924537:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177102.924540:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177103.564468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177103.564478:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177103.564481:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177103.564487:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177103.564492:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177103.564497:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177103.564501:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177103.564504:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177103.564507:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177103.948463:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177103.948467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177103.948471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177103.948472:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177103.948475:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177103.948478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177103.948480:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177103.948481:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177103.948485:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177103.948486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177103.948490:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177103.948490:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177103.948495:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177103.948498:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177103.948500:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177103.948562:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177103.948567:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177103.948571:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786177104.076456:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177104.076467:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786177104.076470:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177104.076474:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177104.076479:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177104.076486:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11778 00000400:00000200:0.0:1786177104.076495:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177104.076501:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177104.076505:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177104.076508:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076511:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177104.076514:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076517:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177104.076521:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177104.076524:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786177104.076527:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177104.076530:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786177104.076533:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786177104.076536:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786177104.076538:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076540:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177104.076542:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076544:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177104.076547:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786177104.076549:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076552:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177104.076555:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177104.076558:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177104.076560:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.076563:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177104.076565:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177104.076574:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786177104.076577:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786177104.076581:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177104.076583:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.076586:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177104.076589:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177104.076594:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177104.076601:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786177104.076607:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177104.076617:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177104.076627:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177104.076631:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177104.076637:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177104.076659:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177104.076662:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177104.076666:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786177104.076671:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177104.076673:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177104.076676:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177104.076679:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11778 00000400:00000200:0.0:1786177104.076682:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177104.076685:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177104.076687:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177104.076690:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076692:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177104.076695:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076697:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177104.076700:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177104.076701:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786177104.076703:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177104.076705:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786177104.076707:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786177104.076709:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786177104.076712:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177104.076712:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177104.076717:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177104.076719:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1786177104.076720:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076724:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177104.076724:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1786177104.076729:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786177104.076733:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076737:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177104.076737:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786177104.076741:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177104.076748:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177104.076752:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1786177104.076753:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.076757:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177104.076757:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1786177104.076762:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1786177104.076763:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:0.0:1786177104.076767:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.076769:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177104.076770:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.076776:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177104.076777:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786177104.076782:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000100:00100000:1.0:1786177104.076784:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235575680:192.168.202.132@tcp:250:kworker.0 02000000:00000001:0.0:1786177104.076786:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177104.076789:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.076793:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177104.076793:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177104.076797:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177104.076800:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.076800:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:0.0:1786177104.076804:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:1.0:1786177104.076805:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.076810:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.076811:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1786177104.076819:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1786177104.076820:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177104.076826:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.076835:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177104.076836:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177104.076839:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177104.076844:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000100:00000040:0.0:1786177104.076845:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1786177104.076854:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000100:00000001:0.0:1786177104.076856:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177104.076861:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1786177104.076864:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235575680, portal 25 00000100:00080000:0.0:1786177104.076865:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177104.076869:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177104.076870:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177104.076874:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215040 : -110371915336576 : ffff9b9e09317480) 00000100:00080000:0.0:1786177104.076875:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177104.076881:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177104.076887:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11778 00000100:00000040:1.0:1786177104.076887:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1786177104.076894:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177104.076901:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177104.076907:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177104.076907:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177104.076915:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177104.076916:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076920:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177104.076924:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177104.076924:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e4c8. 00000100:00000001:0.0:1786177104.076929:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000200:1.0:1786177104.076932:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235575680, offset 0 00000020:00000001:0.0:1786177104.076933:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177104.076936:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786177104.076939:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177104.076940:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786177104.076944:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177104.076946:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000020:00000001:0.0:1786177104.076948:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786177104.076953:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076955:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177104.076957:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:0.0:1786177104.076960:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.076964:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177104.076968:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:1.0:1786177104.076968:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00080000:0.0:1786177104.076975:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:1.0:1786177104.076978:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1786177104.076979:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177104.076983:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1786177104.076986:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 02000000:00000001:0.0:1786177104.076988:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177104.076992:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177104.076993:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1786177104.076995:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177104.076998:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177104.076999:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000008:00000001:0.0:1786177104.077004:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.077009:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177104.077010:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426564:426564:256:4294967295] 192.168.202.132@tcp LPNI seq info [426526:426526:8:4294967295] 00000100:00000010:0.0:1786177104.077014:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786177104.077019:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177104.077019:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:0.0:1786177104.077025:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177104.077026:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.077028:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177104.077029:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177104.077033:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177104.077038:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 02000000:00000010:0.0:1786177104.077040:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000800:00000200:1.0:1786177104.077048:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:0.0:1786177104.077058:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177104.077058:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00100000:0.0:1786177104.077068:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1786177104.077069:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177104.077081:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177104.077085:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177104.077089:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177104.077094:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177104.077102:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177104.077104:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177104.077108:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177104.077109:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.077110:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177104.077114:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1786177104.077118:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177104.077120:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177104.077122:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306454784 00000100:00080000:0.0:1786177104.077127:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11778 00000100:00000010:1.0:1786177104.077131:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c625a0. 00000400:00000200:0.0:1786177104.077133:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177104.077139:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177104.077142:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177104.077142:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000040:0.0:1786177104.077148:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.077152:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177104.077155:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177104.077155:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306454784 00000100:00000001:0.0:1786177104.077157:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177104.077162:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177104.077164:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000010:1.0:1786177104.077164:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c629c0. 00000020:00000001:0.0:1786177104.077166:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177104.077170:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:1.0:1786177104.077171:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1786177104.077173:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786177104.077178:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177104.077178:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1786177104.077186:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177104.077186:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.077191:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177104.077192:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177104.077195:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177104.077198:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177104.077199:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177104.077204:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786177104.077207:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177104.077209:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.077210:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1786177104.077212:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177104.077215:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:0.0:1786177104.077216:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177104.077221:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1786177104.077222:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.077226:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177104.077227:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177104.077245:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177104.077245:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000008:00000001:0.0:1786177104.077250:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.077253:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177104.077254:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1786177104.077258:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786177104.077264:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177104.077269:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177104.077271:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1786177104.077272:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.077274:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.077276:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177104.077279:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177104.077281:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177104.077283:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.077285:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1786177104.077289:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:1.0:1786177104.077290:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.077295:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177104.077297:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575808:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177104.077304:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1786177104.077306:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177104.077308:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177104.077312:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.077316:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1786177104.077321:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000010:1.0:1786177104.077322:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae000. 00000100:00000001:0.0:1786177104.077325:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.077327:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177104.077334:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177104.077336:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177104.077341:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177104.077349:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000001:0.0:1786177104.077352:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177104.077354:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235575808, portal 10 00000100:00000001:1.0:1786177104.077387:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177104.077389:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794203392 : -110371915348224 : ffff9b9e09314700) 00000100:00080000:0.0:1786177104.077396:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1786177104.077398:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1786177104.077403:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786177104.077407:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177104.077411:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177104.077413:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1786177104.077416:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:1.0:1786177104.077421:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00080000:0.0:1786177104.077425:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11778 00000400:00000010:1.0:1786177104.077427:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027edd0. 00000400:00000200:0.0:1786177104.077432:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000200:1.0:1786177104.077433:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235575808, offset 0 00000400:00000200:0.0:1786177104.077438:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177104.077443:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177104.077443:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000040:0.0:1786177104.077450:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177104.077453:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177104.077456:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177104.077461:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000040:0.0:1786177104.077462:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.077469:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177104.077470:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1786177104.077474:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177104.077477:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000400:00000200:1.0:1786177104.077478:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000001:0.0:1786177104.077482:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1786177104.077485:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1786177104.077487:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000400:00000200:1.0:1786177104.077491:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000001:0.0:1786177104.077493:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786177104.077497:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000200:1.0:1786177104.077499:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426565:426565:255:4294967295] 192.168.202.132@tcp LPNI seq info [426527:426527:7:4294967295] 00000100:00000040:0.0:1786177104.077504:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.077509:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177104.077510:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:0.0:1786177104.077514:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177104.077518:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177104.077523:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000400:00000200:1.0:1786177104.077527:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00080000:0.0:1786177104.077531:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177104.077536:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177104.077539:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 02000000:00000001:0.0:1786177104.077540:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177104.077544:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177104.077548:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.077550:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177104.077552:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000800:00000010:1.0:1786177104.077554:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e000. 00000008:00000001:0.0:1786177104.077555:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.077558:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177104.077564:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000010:0.0:1786177104.077565:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786177104.077570:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000800:00000200:1.1:1786177104.077573:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:0.0:1786177104.077575:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177104.077579:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.077581:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.077582:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177104.077586:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177104.077588:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177104.077588:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177104.077595:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:0.0:1786177104.077597:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:1.0:1786177104.077598:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306450948 00000100:00000001:0.0:1786177104.077603:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.077605:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306454784 00000100:00000001:1.0:1786177104.077610:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306450948 00000100:00100000:0.0:1786177104.077613:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177104.077615:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306454784 00000100:00000001:1.0:1786177104.077620:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177104.077626:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.077631:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1786177104.077632:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177104.077637:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177104.077641:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786177104.077645:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177104.077664:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177104.077667:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177104.077669:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.077671:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177104.077675:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1786177104.077676:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11788) 02000000:00000001:1.0:1786177104.077679:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177104.077686:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575936:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177104.077690:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177104.077693:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177104.077695:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.077697:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177104.077702:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af000. 02000000:00000001:1.0:1786177104.077704:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177104.077708:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177104.077712:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177104.077715:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ed48. 00000100:00000200:1.0:1786177104.077718:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235575936, portal 10 00000100:00000001:1.0:1786177104.077721:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177104.077723:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794213248 : -110371915338368 : ffff9b9e09316d80) 00000100:00000040:1.0:1786177104.077729:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177104.077736:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177104.077739:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177104.077743:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e5d8. 00000100:00000200:1.0:1786177104.077745:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235575936, offset 0 00000400:00000200:1.0:1786177104.077750:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177104.077754:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177104.077759:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177104.077762:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177104.077765:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177104.077768:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177104.077771:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177104.077774:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426566:426566:254:4294967295] 192.168.202.132@tcp LPNI seq info [426528:426528:6:4294967295] 00000400:00000200:1.0:1786177104.077778:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177104.077784:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177104.077790:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177104.077794:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142ec00. 00000800:00000200:1.0:1786177104.077798:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177104.077802:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177104.077805:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177104.077809:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177104.077813:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177104.077815:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306450948 00000100:00000001:1.0:1786177104.077818:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306454784 00000100:00000001:1.0:1786177104.077820:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306450948 00000100:00000001:1.0:1786177104.077824:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306454784 00000100:00000001:1.0:1786177104.077827:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177104.077829:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.077832:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.077834:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177104.077837:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177104.077839:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1786177104.077841:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177104.077843:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177104.077844:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177104.077849:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177104.077856:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177104.077858:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.077860:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177104.077862:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177104.077864:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177104.077867:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09315500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576064:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177104.077871:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177104.077873:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177104.077875:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.077879:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177104.077882:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177104.077884:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177104.077887:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177104.077890:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842d68. 00000400:00000010:1.0:1786177104.077895:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177104.077898:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235576064, portal 4 00000100:00000001:1.0:1786177104.077900:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177104.077901:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794206976 : -110371915344640 : ffff9b9e09315500) 00000100:00000040:1.0:1786177104.077907:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177104.077914:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177104.077917:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177104.077919:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e220. 00000100:00000200:1.0:1786177104.077921:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235576064, offset 0 00000400:00000200:1.0:1786177104.077926:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177104.077930:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177104.077934:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177104.077937:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177104.077940:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177104.077942:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177104.077945:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177104.077948:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426567:426567:253:4294967295] 192.168.202.132@tcp LPNI seq info [426529:426529:5:4294967295] 00000400:00000200:1.0:1786177104.077953:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177104.077960:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177104.077964:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177104.077968:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e200. 00000800:00000200:1.0:1786177104.077972:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177104.077976:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177104.077980:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177104.077983:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177104.077987:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177104.077989:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306450948 00000100:00000001:1.0:1786177104.077992:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306454784 00000100:00000001:1.0:1786177104.077995:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306450948 00000100:00000001:1.0:1786177104.077997:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306454784 00000100:00000001:1.0:1786177104.078000:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177104.078002:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.078004:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177104.078008:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177104.078015:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177104.078017:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.078020:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177104.078022:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177104.078024:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177104.078027:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576192:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177104.078031:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177104.078033:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177104.078034:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177104.078036:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177104.078040:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177104.078042:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177104.078045:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177104.078048:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177104.078051:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177104.078054:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235576192, portal 4 00000100:00000001:1.0:1786177104.078057:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177104.078058:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794204288 : -110371915347328 : ffff9b9e09314a80) 00000100:00000040:1.0:1786177104.078064:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177104.078071:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177104.078074:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177104.078076:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ecc0. 00000100:00000200:1.0:1786177104.078079:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235576192, offset 0 00000400:00000200:1.0:1786177104.078084:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177104.078088:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177104.078091:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177104.078094:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177104.078096:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177104.078099:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177104.078102:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177104.078105:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426568:426568:252:4294967295] 192.168.202.132@tcp LPNI seq info [426530:426530:4:4294967295] 00000400:00000200:1.0:1786177104.078109:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177104.078114:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177104.078119:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177104.078123:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177104.078127:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177104.078132:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177104.078134:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177104.078137:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177104.078140:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177104.078142:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306450950 00000100:00000001:1.0:1786177104.078145:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306454784 00000100:00000001:1.0:1786177104.078147:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306450950 00000100:00000001:1.0:1786177104.078150:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306454784 00000100:00000001:1.0:1786177104.078152:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177104.078154:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.078157:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.078159:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177104.078162:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177104.078164:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1786177104.078166:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177104.078168:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177104.078170:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.078172:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177104.588468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177104.588476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177104.588478:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177104.588484:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177104.588488:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177104.588492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177104.588495:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177104.588497:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177104.588500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.972465:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177104.972472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177104.972472:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177104.972474:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177104.972478:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177104.972481:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177104.972485:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177104.972486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177104.972489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177104.972490:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177104.972493:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177104.972495:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177104.972497:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177104.972498:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177104.972502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177104.972606:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177104.972610:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177104.972614:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177105.612477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177105.612485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177105.612488:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177105.612494:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177105.612498:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177105.612501:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177105.612505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177105.612508:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177105.612512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177105.996482:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177105.996489:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177105.996489:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177105.996491:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177105.996494:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177105.996496:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177105.996501:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177105.996502:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177105.996504:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177105.996506:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177105.996508:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177105.996512:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177105.996513:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177105.996514:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177105.996517:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177105.996582:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177105.996587:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177105.996590:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177106.636452:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177106.636459:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177106.636461:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177106.636466:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177106.636469:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177106.636472:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177106.636475:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177106.636477:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177106.636480:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177107.020439:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177107.020443:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177107.020446:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177107.020447:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177107.020448:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177107.020452:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177107.020454:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177107.020455:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177107.020456:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177107.020459:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177107.020460:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177107.020461:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177107.020462:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177107.020465:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.020468:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177107.020503:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177107.020507:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177107.020510:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1786177107.148685:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:2.0:1786177107.148729:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1786177107.148738:0:2382:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177107.148743:0:2382:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:2.0:1786177107.148748:0:2382:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177107.148752:0:2382:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1786177107.148758:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:2.0:1786177107.148761:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:2.0:1786177107.148767:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1786177107.148772:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1786177107.148779:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e4c8 00000400:00000010:2.0:1786177107.148782:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e4c8. 00000100:00000001:2.0:1786177107.148787:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177107.148796:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177107.148826:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177107.148831:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177107.148839:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177107.148842:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1786177107.148852:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:2.0:1786177107.148855:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000400:00000200:2.0:1786177107.148859:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1786177107.148863:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1786177107.148866:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027edd0 00000400:00000010:2.0:1786177107.148869:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027edd0. 00000100:00000001:2.0:1786177107.148872:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177107.148878:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.148881:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177107.148888:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177107.148894:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177107.148900:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177107.148902:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177107.148903:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000100:2.0:1786177107.148908:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000100:00000001:1.0:1786177107.148910:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000800:00000010:2.0:1786177107.148912:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:2.0:1786177107.148918:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:1.0:1786177107.148923:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177104/real 1786177107] req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177107.148925:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1786177107.148932:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e5d8 00000400:00000010:2.0:1786177107.148935:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e5d8. 00000100:00000001:2.0:1786177107.148940:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:1.0:1786177107.148943:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000200:2.0:1786177107.148946:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177107.148947:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177107.148952:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177107.148957:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177107.148961:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:2.0:1786177107.148961:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1786177107.148967:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1786177107.148968:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177107.148976:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177107.148979:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177107.148982:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1786177107.148991:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:2.0:1786177107.148995:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000100:00000200:1.0:1786177107.148998:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177107.148999:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1786177107.149009:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1786177107.149014:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e220 00000400:00000010:2.0:1786177107.149018:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e220. 00000100:00000001:2.0:1786177107.149021:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177107.149025:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149030:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177107.149035:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177107.149038:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1786177107.149041:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1786177107.149044:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149045:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177107.149056:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177107.149057:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177107.149061:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1786177107.149070:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:2.0:1786177107.149077:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:2.0:1786177107.149082:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:1.0:1786177107.149086:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1786177107.149089:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1786177107.149095:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ecc0 00000100:00000040:1.0:1786177107.149098:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1786177107.149098:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ecc0. 00000100:00000001:2.0:1786177107.149103:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1786177107.149112:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000200:2.0:1786177107.149112:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149120:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.149125:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177107.149128:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177107.149129:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1786177107.149132:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:2.0:1786177107.149136:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149138:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177107.149145:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1786177107.149151:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177107.149154:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1786177107.149155:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177107.149160:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177107.149167:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177107.149178:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575808:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177107.149183:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177107.149187:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177104/real 1786177107] req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177107.149196:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177107.149197:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177107.149200:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177107.149218:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ed48 00000400:00000010:1.0:1786177107.149220:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ed48. 00000100:00000001:1.0:1786177107.149223:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177107.149226:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177107.149239:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149245:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177107.149246:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.149248:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177107.149249:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177107.149253:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149261:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177107.149265:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149272:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177107.149275:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.149276:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177107.149278:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.149280:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177107.149282:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177107.149285:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177107.149289:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177107.149292:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177107.149296:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177107.149306:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235575936:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177107.149310:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177107.149314:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177104/real 1786177107] req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177107.149329:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177107.149330:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177107.149333:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177107.149335:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177107.149337:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177107.149341:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177107.149344:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177107.149353:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149387:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177107.149389:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.149392:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177107.149393:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177107.149397:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149404:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177107.149408:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149414:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177107.149417:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.149418:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177107.149420:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.149421:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177107.149423:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177107.149425:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177107.149428:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177107.149430:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177107.149434:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177107.149442:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235575680:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177107.149447:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177107.149450:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177107.149453:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314700 x1872930235575808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149460:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177107.149464:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177107.149470:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177107.149475:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177107.149478:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177107.149481:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177107.149483:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177107.149488:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:1.0:1786177107.149493:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177107.149496:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177107.149498:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314700. 00000100:00000001:1.0:1786177107.149503:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177107.149504:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.149506:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177107.149511:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316d80 x1872930235575936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149518:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177107.149519:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177107.149521:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af000. 02000000:00000001:1.0:1786177107.149523:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177107.149525:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177107.149526:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177107.149528:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177107.149530:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:1.0:1786177107.149533:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177107.149534:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177107.149536:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316d80. 00000100:00000001:1.0:1786177107.149539:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177107.149540:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.149542:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177107.149546:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317480 x1872930235575680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.149568:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177107.149570:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177107.149572:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177107.149575:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177107.149576:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177107.149578:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177107.149580:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177107.149582:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:1.0:1786177107.149585:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177107.149586:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177107.149587:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317480. 00000100:00000001:1.0:1786177107.149590:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177107.149591:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.149594:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177107.149596:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177107.149597:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177107.149599:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177107.149603:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177104/real 1786177107] req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177107.149611:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177107.149613:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177107.149614:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842d68. 00000400:00000200:1.0:1786177107.149617:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177107.149619:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177107.149623:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177107.149627:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177107.149636:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.151274:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177107.151278:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.151281:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177107.151283:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177107.151291:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.151300:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177107.151305:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.151313:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177107.151316:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.151318:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177107.151319:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.151322:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177107.151324:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177107.151327:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177107.151331:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177107.151334:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177107.151338:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177107.151347:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09315500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576064:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177107.151352:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177107.151370:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177104/real 1786177107] req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177107.151381:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177107.151384:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177107.151387:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177107.151392:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177107.151395:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177107.151400:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177107.151404:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177107.151415:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.151422:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177107.151424:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.151426:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177107.151428:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177107.151432:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.151439:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177107.151444:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.151452:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177107.151454:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.151456:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177107.151457:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.151473:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177107.151475:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177107.151478:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177107.151482:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177107.151485:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177107.151490:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177107.151498:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576192:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177107.151503:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.151506:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177107.151510:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09315500 x1872930235576064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.151517:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177107.151519:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177107.151522:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177107.151524:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177107.151527:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177107.151528:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177107.151530:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177107.151532:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:1.0:1786177107.151534:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177107.151535:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177107.151537:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09315500. 00000100:00000001:1.0:1786177107.151539:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177107.151540:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.151541:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177107.151543:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314a80 x1872930235576192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177107.151550:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177107.151552:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177107.151562:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177107.151566:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177107.151568:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177107.151569:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177107.151571:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177107.151573:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:1.0:1786177107.151576:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177107.151577:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177107.151578:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314a80. 00000100:00000001:1.0:1786177107.151581:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177107.151583:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.151585:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177107.151592:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177107.151594:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177107.151597:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177107.151599:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177107.660476:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177107.660484:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177107.660493:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177107.660499:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177107.660503:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177107.660508:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177107.660511:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177107.660513:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177107.660517:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177107.980471:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306454784, at 4306454848 00000100:00000040:1.0:1786177107.980471:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306454784, at 4306454848 00000100:00000010:0.0:1786177107.980486:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c625a0. 00000100:00000010:1.0:1786177107.980489:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c629c0. 00000100:00000001:1.0:1786177108.044448:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177108.044449:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177108.044456:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177108.044457:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177108.044462:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177108.044464:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177108.044467:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177108.044468:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177108.044472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177108.044474:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177108.044476:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177108.044480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177108.044485:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177108.044488:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177108.044491:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177108.044558:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177108.044566:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177108.044571:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177108.684453:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177108.684463:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177108.684466:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177108.684472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177108.684476:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177108.684480:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177108.684483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177108.684486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177108.684489:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.068462:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177109.068466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177109.068469:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.068470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177109.068473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177109.068474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177109.068476:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177109.068477:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177109.068479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177109.068480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.068480:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177109.068484:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.068487:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177109.068489:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.068492:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177109.068537:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177109.068541:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177109.068543:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177109.196540:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177109.196550:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177109.196553:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177109.196557:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177109.196562:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177109.196570:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11783 00000400:00000200:2.0:1786177109.196579:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177109.196595:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177109.196599:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177109.196603:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.196606:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177109.196609:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.196612:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177109.196616:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177109.196618:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177109.196622:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177109.196625:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177109.196628:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177109.196631:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177109.196634:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.196637:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177109.196639:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.196641:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177109.196644:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177109.196648:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177109.196651:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177109.196654:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177109.196657:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177109.196668:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.196672:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177109.196675:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177109.196686:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736d80. 00000020:00000040:2.0:1786177109.196691:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177109.196695:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177109.196697:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.196713:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177109.196717:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177109.196721:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177109.196729:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:2.0:1786177109.196736:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177109.196747:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177109.196759:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177109.196764:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177109.196769:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e4f7686] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177109.196792:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177109.196796:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177109.196799:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177109.196801:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177109.196803:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177109.196807:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177109.196810:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11783 00000400:00000200:2.0:1786177109.196815:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177109.196818:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177109.196821:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177109.196825:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.196828:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177109.196830:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.196833:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177109.196836:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177109.196837:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000100:00000001:1.0:1786177109.196840:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1786177109.196840:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177109.196843:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00000001:1.0:1786177109.196846:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1786177109.196848:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177109.196851:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1786177109.196854:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177109.196858:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.196862:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177109.196868:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177109.196868:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.196874:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177109.196880:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177109.196886:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177109.196892:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177109.196893:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.196897:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177109.196898:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177109.196904:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177109.196905:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177109.196908:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177109.196909:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177109.196914:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.196915:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177109.196921:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1786177109.196923:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235576320:192.168.202.132@tcp:250:kworker.0 00000008:00000001:2.0:1786177109.196927:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.196931:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1786177109.196932:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177109.196939:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737480. 02000000:00000001:1.0:1786177109.196940:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1786177109.196944:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177109.196945:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177109.196950:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177109.196951:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1786177109.196953:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.196957:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177109.196959:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:1.0:1786177109.196960:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:2.0:1786177109.196962:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177109.196965:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177109.196969:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000400:00000200:1.0:1786177109.196975:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177109.196975:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177109.196983:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000100:00100000:2.0:1786177109.196985:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177109.196991:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000001:2.0:1786177109.196999:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177109.197002:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177109.197004:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235576320, portal 25 00000100:00000001:1.0:1786177109.197009:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177109.197011:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469627264 : -110371239924352 : ffff9b9e31736d80) 00000100:00000040:2.0:1786177109.197014:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046bba14a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177109.197021:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177109.197036:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.197040:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1786177109.197042:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1786177109.197048:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00080000:2.0:1786177109.197050:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177109.197053:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:1.0:1786177109.197054:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00080000:2.0:1786177109.197055:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1786177109.197061:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235576320, offset 0 00000100:00000001:2.0:1786177109.197062:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177109.197068:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11783 00000400:00000200:2.0:1786177109.197075:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177109.197077:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177109.197082:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177109.197085:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177109.197090:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177109.197091:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197096:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177109.197099:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197101:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177109.197104:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177109.197105:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177109.197107:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177109.197109:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177109.197111:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177109.197114:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177109.197118:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197120:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177109.197123:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197125:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177109.197128:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177109.197131:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197134:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177109.197136:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177109.197139:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177109.197141:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.197144:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177109.197146:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177109.197149:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.197152:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177109.197156:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734700. 00000020:00000040:2.0:1786177109.197159:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177109.197162:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177109.197164:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.197166:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177109.197167:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177109.197169:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177109.197173:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177109.197177:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177109.197198:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177109.197207:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177109.197209:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177109.197212:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088313762] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177109.197220:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177109.197223:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177109.197226:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177109.197228:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177109.197229:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177109.197232:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177109.197235:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11783 00000400:00000200:2.0:1786177109.197239:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177109.197242:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177109.197244:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177109.197247:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197249:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177109.197250:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197253:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177109.197255:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177109.197256:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177109.197258:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177109.197264:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177109.197266:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177109.197268:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177109.197271:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197273:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177109.197275:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197277:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177109.197280:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177109.197283:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197286:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177109.197288:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177109.197290:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177109.197293:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.197295:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177109.197297:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177109.197300:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.197302:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177109.197305:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000020:00000040:2.0:1786177109.197308:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177109.197310:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177109.197312:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.197314:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177109.197315:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177109.197317:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177109.197320:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000001:2.0:1786177109.197323:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177109.197328:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177109.197342:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177109.197344:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177109.197348:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177109.197378:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177109.197382:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177109.197385:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177109.197387:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177109.197388:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177109.197391:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177109.197394:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11783 00000400:00000200:2.0:1786177109.197397:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177109.197401:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177109.197403:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177109.197405:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197408:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177109.197410:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197412:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177109.197414:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177109.197416:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177109.197417:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177109.197419:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177109.197421:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177109.197423:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177109.197426:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197428:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177109.197430:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197432:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177109.197435:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177109.197438:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177109.197440:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177109.197442:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177109.197446:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177109.197448:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.197450:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177109.197452:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177109.197455:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.197457:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177109.197461:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786177109.197464:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177109.197466:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177109.197468:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177109.197470:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177109.197471:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177109.197473:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177109.197477:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:2.0:1786177109.197480:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177109.197486:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177109.197494:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177109.197496:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177109.197501:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177109.197509:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177109.197514:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11793) 00000400:00000200:1.0:1786177109.198559:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177109.198566:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177109.198571:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177109.198575:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177109.198580:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177109.198584:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426569:426569:256:4294967295] 192.168.202.132@tcp LPNI seq info [426531:426531:8:4294967295] 00000400:00000200:1.0:1786177109.198590:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177109.198601:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177109.198609:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177109.198617:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177109.198624:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177109.198628:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177109.198634:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177109.198639:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177109.198644:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177109.198647:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177109.198653:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177109.198658:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177109.198665:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177109.198668:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177109.198673:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177109.198679:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.198691:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177109.198695:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.198703:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.198705:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177109.198712:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177109.198717:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177109.198721:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306459906 00000100:00000010:1.0:1786177109.198728:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c629c0. 00000100:00000001:1.0:1786177109.198735:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177109.198742:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306459906 00000100:00000010:1.0:1786177109.198746:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62de0. 00000100:00000001:1.0:1786177109.198749:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177109.198752:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177109.198756:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.198759:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177109.198765:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177109/real 1786177109] req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177109.198776:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177109.198778:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177109.198781:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177109.198784:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177109.198794:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177109.198796:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177109.198801:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177109.198808:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.198815:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177109.198816:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.198819:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177109.198821:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177109.198825:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.198833:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177109.198838:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.198845:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177109.198848:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.198850:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177109.198852:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.198854:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177109.198856:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177109.198859:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177109.198863:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177109.198865:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177109.198869:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177109.198877:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235576320:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177109.198883:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.198886:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177109.198889:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736d80 x1872930235576320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.198901:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177109.198904:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177109.198907:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177109.198910:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177109.198912:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177109.198914:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177109.198916:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177109.198919:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:1.0:1786177109.198922:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177109.198924:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177109.198926:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736d80. 00000100:00000001:1.0:1786177109.198930:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177109.198931:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.198933:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.198941:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177109.198944:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.198947:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177109.198949:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177109.198951:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177109.198956:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.198963:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177109.198965:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177109.198968:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177109.198970:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177109.198973:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177109.198977:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576448:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177109.198981:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177109.198985:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177109.198988:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177109.198990:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177109.198994:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177109.198997:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177109.199002:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177109.199006:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177109.199010:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177109.199014:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235576448, portal 10 00000100:00000001:1.0:1786177109.199017:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177109.199018:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469629056 : -110371239922560 : ffff9b9e31737480) 00000100:00000040:1.0:1786177109.199026:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199032:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177109.199036:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177109.199039:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177109.199042:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235576448, offset 0 00000400:00000200:1.0:1786177109.199048:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177109.199052:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177109.199057:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177109.199060:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177109.199062:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177109.199065:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177109.199068:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177109.199071:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426570:426570:256:4294967295] 192.168.202.132@tcp LPNI seq info [426532:426532:8:4294967295] 00000400:00000200:1.0:1786177109.199075:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177109.199082:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177109.199087:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177109.199090:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177109.199101:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177109.199104:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177109.199109:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177109.199112:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177109.199116:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177109.199119:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177109.199122:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177109.199126:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177109.199129:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177109.199132:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177109.199136:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177109.199141:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199150:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177109.199153:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199160:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.199162:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177109.199166:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177109.199170:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177109.199174:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177109.199176:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306456069 00000100:00000001:1.0:1786177109.199179:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306459906 00000100:00000001:1.0:1786177109.199201:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306456069 00000100:00000001:1.0:1786177109.199204:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306459906 00000100:00000001:1.0:1786177109.199206:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177109.199208:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.199210:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177109.199215:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177109/real 1786177109] req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177109.199223:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177109.199225:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177109.199227:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177109.199230:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177109.199232:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177109.199234:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177109.199237:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177109.199246:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199260:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177109.199262:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.199264:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177109.199265:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177109.199269:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199276:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177109.199280:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199287:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177109.199290:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.199291:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177109.199292:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.199294:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177109.199297:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177109.199299:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177109.199302:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177109.199311:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177109.199315:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177109.199323:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576448:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177109.199328:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177109.199332:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199338:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177109.199339:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177109.199342:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177109.199343:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177109.199346:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177109.199348:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576576:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177109.199352:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177109.199354:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177109.199356:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177109.199382:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177109.199387:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177109.199390:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177109.199394:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177109.199398:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177109.199402:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177109.199406:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235576576, portal 10 00000100:00000001:1.0:1786177109.199409:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177109.199411:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469617408 : -110371239934208 : ffff9b9e31734700) 00000100:00000040:1.0:1786177109.199418:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199425:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177109.199429:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177109.199432:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177109.199434:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235576576, offset 0 00000400:00000200:1.0:1786177109.199439:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177109.199443:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177109.199447:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177109.199451:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177109.199454:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177109.199456:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177109.199458:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177109.199461:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426571:426571:256:4294967295] 192.168.202.132@tcp LPNI seq info [426533:426533:8:4294967295] 00000400:00000200:1.0:1786177109.199466:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177109.199473:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177109.199478:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177109.199482:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177109.199487:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177109.199489:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177109.199493:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177109.199496:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177109.199499:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177109.199502:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177109.199505:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177109.199508:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177109.199512:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177109.199514:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177109.199516:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177109.199520:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199529:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177109.199532:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199540:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.199541:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177109.199545:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177109.199548:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177109.199551:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177109.199553:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306456070 00000100:00000001:1.0:1786177109.199556:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306459906 00000100:00000001:1.0:1786177109.199558:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306456070 00000100:00000001:1.0:1786177109.199561:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306459906 00000100:00000001:1.0:1786177109.199563:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177109.199565:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.199567:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177109.199571:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177109/real 1786177109] req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177109.199579:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177109.199580:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177109.199582:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177109.199585:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177109.199586:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177109.199589:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177109.199592:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177109.199600:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199607:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177109.199609:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.199611:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177109.199612:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177109.199616:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199623:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177109.199627:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199634:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177109.199637:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.199639:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177109.199640:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.199642:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177109.199644:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177109.199647:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177109.199650:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177109.199653:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177109.199657:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177109.199666:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576576:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177109.199671:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177109.199675:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199681:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177109.199682:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177109.199685:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177109.199687:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177109.199689:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177109.199692:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576704:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177109.199695:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177109.199697:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177109.199699:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177109.199701:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177109.199704:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177109.199706:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177109.199709:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177109.199712:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177109.199715:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177109.199718:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235576704, portal 4 00000100:00000001:1.0:1786177109.199719:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177109.199721:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469621888 : -110371239929728 : ffff9b9e31735880) 00000100:00000040:1.0:1786177109.199727:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199733:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177109.199735:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177109.199737:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177109.199740:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235576704, offset 0 00000400:00000200:1.0:1786177109.199744:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177109.199748:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177109.199752:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177109.199755:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177109.199758:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177109.199760:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177109.199763:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177109.199765:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426572:426572:256:4294967295] 192.168.202.132@tcp LPNI seq info [426534:426534:8:4294967295] 00000400:00000200:1.0:1786177109.199770:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177109.199776:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177109.199780:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177109.199784:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177109.199787:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177109.199790:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177109.199793:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177109.199796:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177109.199799:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177109.199802:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177109.199804:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177109.199808:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177109.199811:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177109.199813:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177109.199815:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177109.199819:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199826:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177109.199830:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199836:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.199838:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177109.199841:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177109.199845:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177109.199848:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177109.199850:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306456070 00000100:00000001:1.0:1786177109.199853:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306459906 00000100:00000001:1.0:1786177109.199855:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306456070 00000100:00000001:1.0:1786177109.199857:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306459906 00000100:00000001:1.0:1786177109.199859:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177109.199861:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.199863:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177109.199867:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177109/real 1786177109] req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177109.199875:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177109.199876:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177109.199878:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177109.199881:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177109.199882:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177109.199885:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177109.199889:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177109.199898:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199905:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177109.199907:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.199909:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177109.199910:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177109.199914:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199921:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177109.199926:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199933:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177109.199936:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.199938:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177109.199939:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.199941:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177109.199943:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177109.199946:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177109.199950:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177109.199958:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177109.199963:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177109.199971:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576704:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177109.199975:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177109.199978:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.199985:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177109.199987:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177109.199990:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177109.199991:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177109.199993:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177109.199996:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576832:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177109.199999:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177109.200001:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177109.200003:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177109.200005:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177109.200008:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af000. 02000000:00000001:1.0:1786177109.200010:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177109.200014:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177109.200016:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177109.200019:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177109.200023:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235576832, portal 4 00000100:00000001:1.0:1786177109.200025:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177109.200026:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469622784 : -110371239928832 : ffff9b9e31735c00) 00000100:00000040:1.0:1786177109.200032:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.200039:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177109.200041:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177109.200043:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177109.200046:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235576832, offset 0 00000400:00000200:1.0:1786177109.200050:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177109.200055:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177109.200058:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177109.200061:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177109.200064:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177109.200067:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177109.200069:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177109.200072:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426573:426573:256:4294967295] 192.168.202.132@tcp LPNI seq info [426535:426535:8:4294967295] 00000400:00000200:1.0:1786177109.200077:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177109.200084:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177109.200089:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177109.200092:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177109.200096:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177109.200098:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177109.200101:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177109.200103:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177109.200105:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177109.200107:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177109.200109:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177109.200111:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177109.200114:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177109.200115:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177109.200117:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177109.200119:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.200129:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177109.200131:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177134 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.200135:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.200137:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177109.200139:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177109.200141:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177109.200143:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177109.200145:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306456072 00000100:00000001:1.0:1786177109.200146:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306459906 00000100:00000001:1.0:1786177109.200148:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306456072 00000100:00000001:1.0:1786177109.200149:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306459906 00000100:00000001:1.0:1786177109.200151:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177109.200152:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.200153:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177109.200155:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177109/real 1786177109] req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177109.200160:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177109.200161:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177109.200162:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177109.200164:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177109.200165:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177109.200167:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177109.200169:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177109.200174:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.200178:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177109.200179:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.200180:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177109.200212:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177109.200217:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.200224:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177109.200229:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.200240:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177109.200243:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.200244:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177109.200246:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.200248:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177109.200249:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177109.200251:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177109.200255:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177109.200257:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177109.200261:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177109.200269:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235576832:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177109.200273:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.200275:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177109.200279:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737480 x1872930235576448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.200285:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177109.200287:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177109.200290:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177109.200293:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177109.200294:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177109.200296:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177109.200299:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177109.200301:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:1.0:1786177109.200304:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177109.200305:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177109.200307:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737480. 00000100:00000001:1.0:1786177109.200310:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177109.200312:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.200317:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177109.200321:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734700 x1872930235576576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.200328:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177109.200330:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177109.200331:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177109.200333:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177109.200334:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177109.200336:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177109.200337:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177109.200339:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:1.0:1786177109.200341:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177109.200342:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177109.200343:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734700. 00000100:00000001:1.0:1786177109.200346:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177109.200347:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.200349:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177109.200351:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735880 x1872930235576704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.200383:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177109.200385:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177109.200387:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177109.200391:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177109.200392:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177109.200393:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177109.200403:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177109.200405:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:1.0:1786177109.200408:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177109.200410:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177109.200415:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735880. 00000100:00000001:1.0:1786177109.200418:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177109.200419:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.200421:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177109.200424:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735c00 x1872930235576832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177134 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177109.200430:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177109.200431:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177109.200432:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af000. 02000000:00000001:1.0:1786177109.200434:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177109.200435:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177109.200437:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177109.200439:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177109.200440:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:1.0:1786177109.200442:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177109.200443:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177109.200444:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735c00. 00000100:00000001:1.0:1786177109.200446:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177109.200447:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.200449:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177109.200454:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177109.200456:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177109.200458:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177109.200459:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177109.708484:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177109.708492:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177109.708495:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177109.708501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177109.708505:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177109.708509:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177109.708512:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177109.708515:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177109.708518:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177110.092470:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177110.092474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177110.092478:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177110.092480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177110.092483:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177110.092487:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177110.092490:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177110.092492:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177110.092495:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177110.092497:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177110.092501:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177110.092502:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177110.092507:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177110.092508:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177110.092512:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177110.092512:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177110.092518:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177110.092520:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177110.092522:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177110.092528:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177110.092530:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177110.092589:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177110.092592:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177110.092595:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177110.732446:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177110.732455:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177110.732458:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177110.732463:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177110.732468:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177110.732472:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177110.732475:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177110.732478:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177110.732481:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177111.116472:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177111.116478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177111.116479:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177111.116482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177111.116484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177111.116490:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177111.116491:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177111.116495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177111.116496:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177111.116501:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177111.116502:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177111.116506:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177111.116506:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177111.116510:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177111.116514:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177111.116565:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177111.116569:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177111.116572:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177111.756473:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177111.756482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177111.756485:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177111.756491:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177111.756495:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177111.756500:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177111.756504:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177111.756507:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177111.756511:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177112.140505:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177112.140514:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177112.140517:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177112.140523:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177112.140528:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177112.140532:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177112.140532:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177112.140535:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177112.140538:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177112.140540:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177112.140546:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177112.140546:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177112.140550:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177112.140555:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177112.140557:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177112.140725:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177112.140729:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177112.140733:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177112.780426:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177112.780434:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177112.780437:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177112.780443:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177112.780447:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177112.780451:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177112.780455:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177112.780458:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177112.780462:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177113.100474:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306459906, at 4306459968 00000100:00000040:1.0:1786177113.100475:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306459906, at 4306459968 00000100:00000010:0.0:1786177113.100489:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c629c0. 00000100:00000010:1.0:1786177113.100492:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62de0. 00000100:00000001:1.0:1786177113.164478:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177113.164481:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177113.164487:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177113.164488:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177113.164491:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177113.164492:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177113.164497:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177113.164499:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177113.164502:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177113.164505:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177113.164506:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177113.164511:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177113.164512:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177113.164514:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177113.164518:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177113.164521:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177113.164523:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177113.164524:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177113.164529:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177113.164530:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177113.164535:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177113.164537:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177113.164539:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177113.164541:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177113.164542:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177113.164544:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177113.164547:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177113.164596:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177113.164600:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177113.164602:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177113.804468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177113.804476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177113.804479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177113.804485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177113.804490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177113.804493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177113.804496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177113.804499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177113.804502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.188458:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177114.188465:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177114.188466:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177114.188471:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177114.188474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177114.188475:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177114.188480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.188480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177114.188486:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177114.188489:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177114.188490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177114.188492:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177114.188494:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177114.188509:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177114.188512:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177114.188573:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177114.188576:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177114.188580:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177114.316497:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177114.316509:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177114.316513:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177114.316517:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177114.316522:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177114.316530:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11788 00000400:00000200:1.0:1786177114.316539:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177114.316546:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177114.316549:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177114.316553:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177114.316557:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177114.316560:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177114.316563:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177114.316567:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177114.316570:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177114.316573:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177114.316576:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177114.316580:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177114.316583:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177114.316586:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177114.316588:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177114.316591:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177114.316593:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177114.316596:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177114.316600:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177114.316602:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177114.316606:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177114.316609:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177114.316612:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.316615:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177114.316617:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177114.316626:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8380. 00000020:00000040:1.0:1786177114.316631:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177114.316635:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177114.316637:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.316641:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177114.316644:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177114.316648:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177114.316657:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad400. 00000100:00000001:1.0:1786177114.316664:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177114.316674:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177114.316685:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177114.316690:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177114.316696:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002fbc72ce] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177114.316720:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177114.316724:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177114.316728:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177114.316739:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177114.316741:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177114.316745:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177114.316748:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11788 00000400:00000200:1.0:1786177114.316752:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177114.316757:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177114.316757:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1786177114.316760:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177114.316763:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1786177114.316766:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177114.316768:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177114.316770:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177114.316774:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177114.316777:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177114.316781:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:2.0:1786177114.316781:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177114.316785:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177114.316788:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177114.316791:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177114.316796:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177114.316802:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177114.316803:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177114.316807:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177114.316808:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177114.316813:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1786177114.316814:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1786177114.316817:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:2.0:1786177114.316819:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177114.316821:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177114.316826:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 02000000:00000001:2.0:1786177114.316826:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177114.316834:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00100000:2.0:1786177114.316838:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235576960:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177114.316839:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177114.316844:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177114.316849:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177114.316852:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177114.316854:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177114.316856:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177114.316859:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.316861:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:2.0:1786177114.316866:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1786177114.316868:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177114.316874:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177114.316874:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177114.316880:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.316881:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177114.316889:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9880. 00000400:00000200:2.0:1786177114.316890:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:1.0:1786177114.316895:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000010:2.0:1786177114.316897:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 02000000:00000001:1.0:1786177114.316900:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177114.316903:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.316905:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177114.316907:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1786177114.316909:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 02000000:00000001:1.0:1786177114.316910:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177114.316918:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af800. 00000100:00000200:2.0:1786177114.316920:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235576960, portal 25 00000100:00000001:1.0:1786177114.316923:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.316925:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177114.316929:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982976 : -110371221568640 : ffff9b9e328b8380) 00000100:00100000:1.0:1786177114.316934:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1786177114.316940:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177114.316956:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177114.316960:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1786177114.316962:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177114.316968:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000040:1.0:1786177114.316973:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2b2226e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1786177114.316974:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177114.316982:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235576960, offset 0 00000400:00000200:2.0:1786177114.316995:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177114.317007:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177114.317009:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177114.317013:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1786177114.317020:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00080000:1.0:1786177114.317021:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177114.317026:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177114.317030:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1786177114.317030:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1786177114.317037:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177114.317039:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00080000:1.0:1786177114.317044:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11788 00000400:00000200:2.0:1786177114.317050:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177114.317054:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177114.317058:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177114.317063:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177114.317068:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177114.317069:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426574:426574:256:4294967295] 192.168.202.132@tcp LPNI seq info [426536:426536:8:4294967295] 00000100:00000040:1.0:1786177114.317075:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177114.317080:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177114.317083:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177114.317086:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177114.317090:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177114.317094:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177114.317097:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177114.317101:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:2.0:1786177114.317103:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000020:00000040:1.0:1786177114.317105:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177114.317112:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177114.317117:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000200:2.0:1786177114.317118:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000040:1.0:1786177114.317125:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000010:2.0:1786177114.317127:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00000001:1.0:1786177114.317130:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1786177114.317164:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:1.0:1786177114.317170:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:2.0:1786177114.317172:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:1.0:1786177114.317175:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177114.317181:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000200:2.1:1786177114.317186:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:1.0:1786177114.317189:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:2.1:1786177114.317194:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:1.0:1786177114.317195:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177114.317201:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:2.0:1786177114.317206:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177114.317210:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 02000000:00000001:1.0:1786177114.317216:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1786177114.317217:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1786177114.317224:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177114.317228:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1786177114.317230:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177114.317235:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1786177114.317238:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000008:00000001:1.0:1786177114.317240:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177114.317243:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:1.0:1786177114.317245:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.317251:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:1.0:1786177114.317253:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9500. 00000020:00000040:1.0:1786177114.317258:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000200:2.0:1786177114.317259:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177114.317263:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177114.317266:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.317271:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177114.317275:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177114.317279:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.317280:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:1.0:1786177114.317285:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae000. 00000100:00000040:2.0:1786177114.317287:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177114.317293:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.317302:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177114.317305:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177114.317306:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177114.317314:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.317317:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177114.317322:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177114.317323:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177114.317329:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306465024 00000100:00000040:1.0:1786177114.317330:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000cd924b3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1786177114.317339:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290ccc0. 00000100:00000001:1.0:1786177114.317343:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177114.317347:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786177114.317349:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1786177114.317353:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177114.317383:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306465025 00000100:00000010:2.0:1786177114.317390:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c060. 00000100:00000001:2.0:1786177114.317395:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177114.317400:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177114.317401:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1786177114.317404:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177114.317406:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177114.317409:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1786177114.317412:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:2.0:1786177114.317418:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177114/real 1786177114] req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177114.317419:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11788 00000400:00000200:1.0:1786177114.317430:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177114.317439:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000001:2.0:1786177114.317442:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:1.0:1786177114.317444:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000001:2.0:1786177114.317448:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:1.0:1786177114.317451:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:2.0:1786177114.317453:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000100:00000001:1.0:1786177114.317455:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177114.317459:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000100:00000040:1.0:1786177114.317461:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:2.0:1786177114.317465:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:1.0:1786177114.317466:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177114.317472:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177114.317473:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:1.0:1786177114.317474:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177114.317478:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177114.317481:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000200:2.0:1786177114.317482:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177114.317485:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177114.317491:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:1.0:1786177114.317500:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177114.317506:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1786177114.317507:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177114.317511:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177114.317517:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177114.317523:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:2.0:1786177114.317528:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:1.0:1786177114.317529:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177114.317532:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177114.317535:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.317537:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1786177114.317540:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177114.317541:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177114.317546:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1786177114.317551:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177114.317552:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.317558:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177114.317562:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1786177114.317568:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1786177114.317569:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.317575:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177114.317580:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1786177114.317581:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8000. 00000020:00000040:1.0:1786177114.317586:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177114.317593:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177114.317596:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.317599:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177114.317600:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177114.317604:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1786177114.317606:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177114.317608:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.317611:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000010:1.0:1786177114.317615:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261acc00. 00000100:00000001:2.0:1786177114.317615:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.317619:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177114.317622:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.317623:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177114.317630:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1786177114.317633:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1786177114.317638:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177114.317645:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1786177114.317653:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1786177114.317655:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177114.317658:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177114.317669:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ab5f277] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177114.317675:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235576960:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177114.317686:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.317691:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1786177114.317692:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177114.317696:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8380 x1872930235576960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177114.317698:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177114.317710:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177114.317715:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1786177114.317717:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:1.0:1786177114.317719:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1786177114.317721:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1786177114.317726:0:191280:0:(import.c:485:import_select_connection()) Process entered 02000000:00000010:2.0:1786177114.317727:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177114.317732:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1786177114.317733:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11788 00000020:00000001:2.0:1786177114.317738:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:1.0:1786177114.317742:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:2.0:1786177114.317743:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177114.317749:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000400:00000200:1.0:1786177114.317750:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 02000000:00000010:2.0:1786177114.317755:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad400. 00000100:00080000:1.0:1786177114.317757:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1786177114.317763:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:1.0:1786177114.317765:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:2.0:1786177114.317767:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1786177114.317770:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:2.0:1786177114.317772:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8380. 00000100:00000040:1.0:1786177114.317777:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177114.317779:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177114.317782:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177114.317783:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1786177114.317790:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177114.317790:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177114.317794:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177114.317798:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177114.317800:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:2.0:1786177114.317801:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1786177114.317804:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177114.317806:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177114.317810:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177114.317811:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177114.317815:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1786177114.317819:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177114.317819:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177114.317824:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177114.317829:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177114.317832:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177114.317838:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177114.317844:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:2.0:1786177114.317847:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177114.317851:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177114.317852:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:2.0:1786177114.317856:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1786177114.317858:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177114.317860:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177114.317864:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177114.317865:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177114.317870:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1786177114.317871:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577088:192.168.202.132@tcp:38:kworker.0 02000000:00000001:1.0:1786177114.317876:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.317879:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1786177114.317881:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:2.0:1786177114.317884:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:1.0:1786177114.317886:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 02000000:00000001:2.0:1786177114.317888:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1786177114.317892:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177114.317893:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1786177114.317896:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177114.317900:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000010:1.0:1786177114.317903:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bbb80. 02000000:00000001:2.0:1786177114.317904:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177114.317911:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000400:00000200:2.0:1786177114.317915:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177114.317918:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:2.0:1786177114.317921:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 02000000:00000001:1.0:1786177114.317922:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177114.317928:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1786177114.317930:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 02000000:00000001:1.0:1786177114.317931:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177114.317935:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177114.317937:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235577088, portal 10 00000100:00000001:2.0:1786177114.317941:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177114.317943:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487988352 : -110371221563264 : ffff9b9e328b9880) 02000000:00000010:1.0:1786177114.317944:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af400. 00000100:00000001:1.0:1786177114.317951:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177114.317955:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177114.317961:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177114.317971:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177114.317979:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177114.317982:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1786177114.317985:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000001:1.0:1786177114.317986:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177114.317991:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235577088, offset 0 00000100:00000040:1.0:1786177114.317997:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3691ff3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177114.318003:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177114.318013:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177114.318019:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177114.318022:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000040:1.0:1786177114.318027:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11798) 00000400:00000200:2.0:1786177114.318028:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177114.318034:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177114.318038:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177114.318042:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177114.318047:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426575:426575:256:4294967295] 192.168.202.132@tcp LPNI seq info [426537:426537:8:4294967295] 00000400:00000200:2.0:1786177114.318052:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177114.318060:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177114.318065:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177114.318069:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177114.318073:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177114.318077:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786177114.318081:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177114.318084:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1786177114.318089:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177114.318091:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177114.318094:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177114.318098:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177114.318102:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177114.318104:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177114.318106:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177114.318111:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318119:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177114.318123:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318130:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.318147:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177114.318151:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177114.318156:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177114.318160:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177114.318162:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306461188 00000100:00000001:2.0:1786177114.318165:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306465024 00000100:00000001:2.0:1786177114.318167:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306461188 00000100:00000001:2.0:1786177114.318169:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306465025 00000100:00000001:2.0:1786177114.318172:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177114.318174:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.318176:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177114.318181:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177114/real 1786177114] req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177114.318190:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177114.318191:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177114.318193:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177114.318196:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177114.318198:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177114.318213:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177114.318222:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177114.318231:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318237:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177114.318239:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.318240:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177114.318242:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177114.318246:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318252:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177114.318256:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318263:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177114.318266:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.318268:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177114.318269:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.318271:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177114.318274:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177114.318277:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177114.318281:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177114.318283:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177114.318288:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177114.318296:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577088:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177114.318301:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177114.318304:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318311:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177114.318312:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177114.318314:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177114.318317:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177114.318319:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177114.318322:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577216:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177114.318325:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177114.318327:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177114.318329:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177114.318331:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177114.318335:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786177114.318337:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177114.318340:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177114.318343:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177114.318347:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177114.318350:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235577216, portal 10 00000100:00000001:2.0:1786177114.318352:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177114.318353:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487987456 : -110371221564160 : ffff9b9e328b9500) 00000100:00000040:2.0:1786177114.318379:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318392:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177114.318395:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177114.318398:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177114.318401:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235577216, offset 0 00000400:00000200:2.0:1786177114.318405:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177114.318410:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177114.318413:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177114.318417:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177114.318419:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177114.318422:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177114.318424:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177114.318427:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426576:426576:256:4294967295] 192.168.202.132@tcp LPNI seq info [426538:426538:8:4294967295] 00000400:00000200:2.0:1786177114.318432:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177114.318438:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177114.318443:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177114.318446:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177114.318450:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177114.318453:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786177114.318457:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177114.318461:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1786177114.318464:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177114.318466:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177114.318469:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177114.318473:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177114.318476:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177114.318478:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177114.318480:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177114.318484:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318493:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177114.318496:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318509:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.318511:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177114.318514:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177114.318517:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177114.318521:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177114.318523:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306461189 00000100:00000001:2.0:1786177114.318526:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306465024 00000100:00000001:2.0:1786177114.318528:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306461189 00000100:00000001:2.0:1786177114.318530:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306465025 00000100:00000001:2.0:1786177114.318533:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177114.318535:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.318537:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177114.318541:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177114/real 1786177114] req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177114.318549:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177114.318551:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177114.318552:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177114.318555:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177114.318557:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177114.318559:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177114.318563:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177114.318572:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318578:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177114.318580:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.318582:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177114.318584:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177114.318588:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318596:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177114.318600:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318607:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177114.318610:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.318611:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177114.318613:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.318615:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177114.318617:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177114.318621:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177114.318624:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177114.318627:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177114.318631:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177114.318639:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577216:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177114.318644:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177114.318648:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318655:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177114.318656:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177114.318659:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177114.318661:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177114.318663:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177114.318667:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577344:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177114.318670:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177114.318672:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177114.318674:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177114.318676:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177114.318680:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177114.318682:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177114.318686:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177114.318689:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177114.318692:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177114.318695:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235577344, portal 4 00000100:00000001:2.0:1786177114.318697:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177114.318698:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982080 : -110371221569536 : ffff9b9e328b8000) 00000100:00000040:2.0:1786177114.318705:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318712:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177114.318714:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177114.318717:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177114.318719:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235577344, offset 0 00000400:00000200:2.0:1786177114.318724:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177114.318728:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177114.318732:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177114.318736:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177114.318738:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177114.318741:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177114.318743:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177114.318747:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426577:426577:256:4294967295] 192.168.202.132@tcp LPNI seq info [426539:426539:8:4294967295] 00000400:00000200:2.0:1786177114.318751:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177114.318758:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177114.318763:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177114.318766:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177114.318771:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177114.318774:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786177114.318778:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177114.318781:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1786177114.318785:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177114.318788:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177114.318791:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177114.318794:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177114.318797:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177114.318799:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177114.318802:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177114.318806:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318814:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177114.318817:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318824:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.318826:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177114.318829:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177114.318832:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177114.318836:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177114.318838:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306461189 00000100:00000001:2.0:1786177114.318841:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306465024 00000100:00000001:2.0:1786177114.318843:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306461189 00000100:00000001:2.0:1786177114.318845:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306465025 00000100:00000001:2.0:1786177114.318848:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177114.318850:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.318852:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177114.318856:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177114/real 1786177114] req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177114.318864:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177114.318866:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177114.318868:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177114.318871:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177114.318872:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177114.318875:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177114.318878:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177114.318886:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318892:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177114.318894:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.318895:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177114.318897:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177114.318901:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318908:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177114.318912:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318919:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177114.318922:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.318923:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177114.318925:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.318927:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177114.318928:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177114.318931:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177114.318934:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177114.318937:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177114.318941:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177114.318949:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577344:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177114.318953:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.318955:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177114.318960:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9880 x1872930235577088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.318966:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177114.318968:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177114.318970:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177114.318973:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177114.318975:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177114.318976:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177114.318978:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177114.318980:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af800. 02000000:00000001:2.0:1786177114.318983:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177114.318984:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177114.318986:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9880. 00000100:00000001:2.0:1786177114.318989:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177114.318991:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.318992:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177114.318996:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9500 x1872930235577216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.319003:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177114.319004:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177114.319006:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786177114.319008:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177114.319010:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177114.319011:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177114.319013:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177114.319015:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:2.0:1786177114.319017:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177114.319018:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177114.319020:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9500. 00000100:00000001:2.0:1786177114.319022:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177114.319023:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.319026:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177114.319029:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8000 x1872930235577344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.319036:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177114.319038:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177114.319040:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177114.319042:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177114.319043:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177114.319045:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177114.319047:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177114.319048:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:2.0:1786177114.319050:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177114.319052:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177114.319053:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8000. 00000100:00000001:2.0:1786177114.319056:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177114.319057:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.319059:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.319063:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177114.319065:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.319067:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177114.319068:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177114.319070:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177114.319074:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.319082:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177114.319083:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177114.319086:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177114.319087:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177114.319089:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177114.319092:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577472:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177114.319096:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177114.319098:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177114.319100:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177114.319102:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177114.319105:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177114.319107:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177114.319110:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177114.319113:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177114.319117:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177114.319120:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235577472, portal 4 00000100:00000001:2.0:1786177114.319123:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177114.319125:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487997312 : -110371221554304 : ffff9b9e328bbb80) 00000100:00000040:2.0:1786177114.319146:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.319154:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177114.319157:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177114.319160:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177114.319163:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235577472, offset 0 00000400:00000200:2.0:1786177114.319167:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177114.319172:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177114.319176:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177114.319180:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177114.319183:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177114.319185:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177114.319188:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177114.319191:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426578:426578:256:4294967295] 192.168.202.132@tcp LPNI seq info [426540:426540:8:4294967295] 00000400:00000200:2.0:1786177114.319196:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177114.319203:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177114.319208:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177114.319212:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177114.319216:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177114.319220:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1786177114.319224:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177114.319227:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1786177114.319231:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177114.319233:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177114.319237:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177114.319240:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177114.319243:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177114.319245:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177114.319248:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177114.319252:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.319267:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177114.319270:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177139 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.319277:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.319279:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177114.319282:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177114.319285:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177114.319289:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177114.319291:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306461191 00000100:00000001:2.0:1786177114.319294:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306465024 00000100:00000001:2.0:1786177114.319296:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306461191 00000100:00000001:2.0:1786177114.319298:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306465025 00000100:00000001:2.0:1786177114.319300:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177114.319301:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.319303:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177114.319307:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177114/real 1786177114] req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177114.319314:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177114.319315:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177114.319316:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177114.319319:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177114.319320:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177114.319323:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177114.319326:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177114.319335:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.319341:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177114.319344:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.319346:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177114.319347:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177114.319351:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.319378:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177114.319383:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.319390:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177114.319393:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.319395:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177114.319396:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.319399:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177114.319400:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177114.319403:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177114.319407:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177114.319409:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177114.319413:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177114.319422:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577472:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177114.319426:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.319428:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177114.319432:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bbb80 x1872930235577472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177139 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177114.319438:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177114.319440:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177114.319442:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177114.319445:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177114.319447:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177114.319449:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177114.319451:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177114.319452:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af400. 02000000:00000001:2.0:1786177114.319455:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177114.319456:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177114.319458:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bbb80. 00000100:00000001:2.0:1786177114.319461:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177114.319463:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.319465:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177114.319469:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177114.319470:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177114.319472:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177114.319474:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177114.828434:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177114.828443:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177114.828446:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177114.828452:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177114.828457:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177114.828461:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177114.828464:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177114.828467:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177114.828470:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177115.212478:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177115.212482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177115.212486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177115.212487:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177115.212491:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177115.212492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177115.212494:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177115.212499:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177115.212499:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177115.212501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177115.212506:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177115.212510:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177115.212513:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177115.212516:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177115.212520:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177115.212563:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177115.212567:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177115.212569:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177115.852492:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177115.852502:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177115.852505:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177115.852512:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177115.852516:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177115.852521:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177115.852524:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177115.852527:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177115.852531:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177116.236454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177116.236461:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177116.236463:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177116.236464:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177116.236468:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177116.236469:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177116.236474:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177116.236476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177116.236478:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177116.236479:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177116.236484:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177116.236485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177116.236486:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177116.236488:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177116.236490:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177116.236522:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177116.236527:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177116.236531:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177116.876463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177116.876469:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177116.876472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177116.876476:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177116.876481:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177116.876484:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177116.876486:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177116.876488:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177116.876490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177117.260474:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177117.260479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177117.260483:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177117.260484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177117.260488:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177117.260490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177117.260493:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177117.260495:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177117.260496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177117.260498:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177117.260503:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177117.260507:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177117.260511:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177117.260513:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177117.260517:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177117.260572:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177117.260577:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177117.260581:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177117.900498:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177117.900504:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177117.900507:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177117.900513:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177117.900517:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177117.900530:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177117.900534:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177117.900537:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177117.900541:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177118.220483:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306465024, at 4306465088 00000100:00000040:1.0:1786177118.220483:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306465025, at 4306465088 00000100:00000010:1.0:1786177118.220499:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c060. 00000100:00000010:0.0:1786177118.220501:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290ccc0. 00000100:00000001:1.0:1786177118.284431:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177118.284437:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177118.284438:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177118.284442:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177118.284443:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177118.284449:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177118.284450:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177118.284453:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177118.284455:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177118.284457:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177118.284459:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177118.284460:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177118.284465:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177118.284468:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177118.284471:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177118.284512:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177118.284525:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177118.284529:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177118.924550:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177118.924556:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177118.924558:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177118.924562:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177118.924564:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177118.924567:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177118.924569:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177118.924571:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177118.924573:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.308451:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177119.308457:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177119.308457:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177119.308460:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177119.308463:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177119.308465:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177119.308469:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177119.308470:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177119.308472:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177119.308475:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.308475:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177119.308478:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177119.308482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177119.308482:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177119.308486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177119.308541:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177119.308544:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177119.308548:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177119.436488:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177119.436498:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177119.436501:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177119.436504:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177119.436509:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177119.436516:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11793 00000400:00000200:1.0:1786177119.436525:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177119.436530:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177119.436534:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177119.436536:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.436538:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177119.436540:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.436541:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177119.436544:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177119.436546:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177119.436547:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177119.436550:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177119.436551:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177119.436553:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177119.436555:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.436556:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177119.436557:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.436558:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177119.436560:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177119.436562:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177119.436564:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177119.436565:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177119.436568:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177119.436570:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.436571:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177119.436573:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177119.436577:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bad80. 00000020:00000040:1.0:1786177119.436580:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177119.436582:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177119.436584:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.436586:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177119.436588:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177119.436591:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177119.436595:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac000. 00000100:00000001:1.0:1786177119.436600:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177119.436607:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177119.436614:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177119.436617:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177119.436619:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b748f214] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177119.436635:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177119.436637:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177119.436639:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177119.436640:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177119.436641:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177119.436644:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177119.436647:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11793 00000400:00000200:1.0:1786177119.436651:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177119.436654:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177119.436656:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1786177119.436657:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177119.436661:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1786177119.436663:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177119.436665:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177119.436669:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177119.436672:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177119.436675:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.436680:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177119.436685:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1786177119.436686:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177119.436687:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177119.436690:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 02000000:00000001:2.0:1786177119.436692:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1786177119.436693:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:2.0:1786177119.436695:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1786177119.436696:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 02000000:00000001:2.0:1786177119.436697:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177119.436702:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177119.436705:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00100000:2.0:1786177119.436705:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235577600:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177119.436711:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1786177119.436712:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.436714:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1786177119.436715:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1786177119.436717:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:2.0:1786177119.436719:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.436720:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177119.436724:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 02000000:00000001:2.0:1786177119.436726:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1786177119.436727:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177119.436729:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177119.436730:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:2.0:1786177119.436732:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 02000000:00000001:1.0:1786177119.436735:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177119.436736:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177119.436738:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.436741:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177119.436744:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1786177119.436746:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000008:00000001:1.0:1786177119.436747:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.436751:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177119.436753:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00000010:1.0:1786177119.436754:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328baa00. 00000020:00000040:1.0:1786177119.436758:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177119.436759:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177119.436760:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.436761:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177119.436762:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1786177119.436763:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 02000000:00000001:1.0:1786177119.436764:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177119.436773:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235577600, portal 25 00000100:00000001:2.0:1786177119.436775:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177119.436777:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487993728 : -110371221557888 : ffff9b9e328bad80) 02000000:00000010:1.0:1786177119.436778:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad000. 00000100:00000001:1.0:1786177119.436783:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177119.436783:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177119.436792:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177119.436793:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177119.436799:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177119.436803:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000001:1.0:1786177119.436804:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177119.436807:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177119.436807:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235577600, offset 0 00000100:00000040:1.0:1786177119.436811:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ebce398] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177119.436817:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177119.436823:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177119.436825:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1786177119.436827:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177119.436832:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:2.0:1786177119.436833:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:1.0:1786177119.436834:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177119.436837:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177119.436840:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177119.436840:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1786177119.436844:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11793 00000400:00000200:2.0:1786177119.436847:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177119.436850:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177119.436852:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177119.436855:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:2.0:1786177119.436857:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00080000:1.0:1786177119.436860:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177119.436862:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426579:426579:256:4294967295] 192.168.202.132@tcp LPNI seq info [426541:426541:8:4294967295] 00000100:00000040:1.0:1786177119.436867:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177119.436868:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177119.436873:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177119.436877:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177119.436882:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:1.0:1786177119.436883:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177119.436888:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:2.0:1786177119.436889:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000020:00000040:1.0:1786177119.436891:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000800:00000010:2.0:1786177119.436893:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000020:00000001:1.0:1786177119.436895:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177119.436899:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000800:00000200:2.0:1786177119.436902:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:1.0:1786177119.436903:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177119.436908:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000200:2.0:1786177119.436909:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000100:00000040:1.0:1786177119.436913:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.436918:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1786177119.436919:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177119.436924:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177119.436930:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000040:1.0:1786177119.436931:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000010:2.0:1786177119.436935:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000100:00000001:1.0:1786177119.436936:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177119.436941:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:2.0:1786177119.436942:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:1.0:1786177119.436946:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:2.0:1786177119.436951:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1786177119.436952:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177119.436955:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1786177119.436958:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 02000000:00000001:1.0:1786177119.436959:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177119.436962:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177119.436962:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:1.0:1786177119.436967:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177119.436969:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1786177119.436969:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:1.0:1786177119.436972:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177119.436975:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177119.436976:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177119.436980:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8a80. 00000020:00000040:1.0:1786177119.436985:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177119.436988:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177119.436989:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1786177119.436990:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.436993:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177119.436995:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1786177119.436996:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177119.436997:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177119.437001:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac800. 00000100:00000001:1.0:1786177119.437006:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437012:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177119.437014:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177119.437016:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177119.437022:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177119.437023:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.437024:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177119.437030:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002bb61190] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177119.437033:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177119.437040:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306470144 00000100:00000001:1.0:1786177119.437041:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177119.437045:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177119.437048:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000010:2.0:1786177119.437048:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c3c0. 00000100:00000001:1.0:1786177119.437052:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177119.437054:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177119.437056:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786177119.437056:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1786177119.437061:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11793 00000100:00000001:2.0:1786177119.437063:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306470144 00000400:00000200:1.0:1786177119.437066:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000010:2.0:1786177119.437068:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cd80. 00000400:00000200:1.0:1786177119.437071:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177119.437072:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1786177119.437073:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177119.437077:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1786177119.437078:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.437093:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177119.437093:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177119.437097:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177119.437098:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1786177119.437100:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177119.437102:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177119.437104:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00100000:2.0:1786177119.437105:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177119/real 1786177119] req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177119.437106:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177119.437109:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177119.437113:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177119.437117:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000001:2.0:1786177119.437121:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1786177119.437122:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000001:2.0:1786177119.437125:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1786177119.437126:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1786177119.437129:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000100:00000040:1.0:1786177119.437130:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.437134:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177119.437134:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177119.437139:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000020:00000040:1.0:1786177119.437140:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:2.0:1786177119.437144:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:1.0:1786177119.437146:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000200:2.0:1786177119.437151:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177119.437152:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177119.437156:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177119.437159:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177119.437161:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.437163:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177119.437166:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177119.437172:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.437173:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177119.437174:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1786177119.437176:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9180. 00000020:00000040:1.0:1786177119.437181:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177119.437186:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177119.437187:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1786177119.437189:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437190:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177119.437191:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1786177119.437193:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1786177119.437194:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1786177119.437195:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177119.437196:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177119.437200:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177119.437201:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac400. 00000100:00000001:1.0:1786177119.437206:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437209:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177119.437215:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1786177119.437216:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437232:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177119.437236:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177119.437239:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177119.437239:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177119.437241:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177119.437244:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437245:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177119.437247:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177119.437250:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177119.437253:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177119.437255:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177119.437257:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177119.437260:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d01dc3b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177119.437271:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235577600:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177119.437275:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437278:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1786177119.437279:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786177119.437281:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1786177119.437283:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000040:2.0:1786177119.437285:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bad80 x1872930235577600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177119.437286:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177119.437288:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177119.437293:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177119.437298:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11793 00000100:00000001:2.0:1786177119.437298:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177119.437302:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:1.0:1786177119.437304:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:2.0:1786177119.437305:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 00000400:00000200:1.0:1786177119.437308:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 02000000:00000001:2.0:1786177119.437309:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1786177119.437311:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000020:00000001:2.0:1786177119.437312:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177119.437314:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000040:1.0:1786177119.437315:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000020:00000001:2.0:1786177119.437317:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:1.0:1786177119.437319:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:2.0:1786177119.437321:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac000. 00000100:00000040:1.0:1786177119.437322:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.437325:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:2.0:1786177119.437325:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:1.0:1786177119.437327:0:191280:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:2.0:1786177119.437328:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000040:1.0:1786177119.437329:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000010:2.0:1786177119.437330:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bad80. 00000020:00000001:1.0:1786177119.437333:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177119.437335:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:2.0:1786177119.437336:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:1.0:1786177119.437337:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177119.437339:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1786177119.437340:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177119.437342:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177119.437345:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.437347:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177119.437348:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.437349:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177119.437350:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:1.0:1786177119.437351:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:2.0:1786177119.437352:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437372:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177119.437374:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177119.437375:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177119.437378:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177119.437382:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177119.437385:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177119.437387:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437388:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177119.437394:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177119.437394:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177119.437397:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177119.437399:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177119.437400:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1786177119.437402:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577728:192.168.202.132@tcp:38:kworker.0 02000000:00000001:1.0:1786177119.437405:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437406:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177119.437408:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1786177119.437410:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:2.0:1786177119.437411:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177119.437414:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1786177119.437415:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 02000000:00000010:2.0:1786177119.437417:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 00000008:00000001:1.0:1786177119.437422:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177119.437422:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177119.437428:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177119.437428:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177119.437431:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00000010:1.0:1786177119.437435:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb800. 00000400:00000010:2.0:1786177119.437436:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177119.437441:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235577728, portal 10 00000020:00000040:1.0:1786177119.437442:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000001:2.0:1786177119.437444:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177119.437447:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487992832 : -110371221558784 : ffff9b9e328baa00) 02000000:00000001:1.0:1786177119.437449:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177119.437451:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177119.437453:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177119.437455:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177119.437459:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1786177119.437462:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1786177119.437464:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177119.437465:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177119.437467:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177119.437470:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235577728, offset 0 02000000:00000010:1.0:1786177119.437471:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae800. 00000400:00000200:2.0:1786177119.437474:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177119.437479:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177119.437479:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177119.437485:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00100000:1.0:1786177119.437489:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786177119.437489:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177119.437493:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177119.437497:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177119.437500:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:1.0:1786177119.437507:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177119.437510:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177119.437514:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426580:426580:256:4294967295] 192.168.202.132@tcp LPNI seq info [426542:426542:8:4294967295] 00000100:00000040:1.0:1786177119.437519:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008148c2a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177119.437524:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177119.437536:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177119.437538:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000040:1.0:1786177119.437541:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11804) 00000800:00000200:2.0:1786177119.437546:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177119.437550:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177119.437555:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177119.437557:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177119.437561:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177119.437562:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177119.437565:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177119.437566:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177119.437574:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177119.437576:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177119.437578:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177119.437580:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177119.437581:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177119.437584:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437593:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177119.437594:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437598:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437599:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177119.437601:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177119.437604:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177119.437606:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177119.437607:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306466308 00000100:00000001:2.0:1786177119.437609:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306470144 00000100:00000001:2.0:1786177119.437610:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306466308 00000100:00000001:2.0:1786177119.437612:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306470144 00000100:00000001:2.0:1786177119.437613:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177119.437614:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437615:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177119.437617:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177119/real 1786177119] req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177119.437621:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177119.437622:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177119.437623:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177119.437624:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177119.437625:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177119.437627:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177119.437628:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177119.437633:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437636:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177119.437637:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.437639:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177119.437639:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177119.437641:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437645:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177119.437647:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437651:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177119.437652:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.437653:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177119.437654:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.437655:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177119.437656:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177119.437657:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177119.437659:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177119.437660:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177119.437662:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177119.437667:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577728:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177119.437669:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177119.437670:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437674:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177119.437675:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177119.437676:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177119.437677:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177119.437678:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177119.437680:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577856:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177119.437682:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177119.437683:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177119.437685:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177119.437686:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177119.437688:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177119.437689:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177119.437691:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177119.437692:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177119.437694:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177119.437696:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235577856, portal 10 00000100:00000001:2.0:1786177119.437697:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177119.437698:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487984768 : -110371221566848 : ffff9b9e328b8a80) 00000100:00000040:2.0:1786177119.437701:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437705:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177119.437706:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177119.437707:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177119.437708:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235577856, offset 0 00000400:00000200:2.0:1786177119.437710:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177119.437713:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177119.437715:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177119.437717:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177119.437718:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177119.437720:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177119.437722:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177119.437725:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426581:426581:256:4294967295] 192.168.202.132@tcp LPNI seq info [426543:426543:8:4294967295] 00000400:00000200:2.0:1786177119.437729:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177119.437734:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177119.437738:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177119.437741:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177119.437744:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177119.437747:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177119.437751:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177119.437753:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177119.437757:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177119.437759:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177119.437761:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177119.437764:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177119.437767:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177119.437768:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177119.437770:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177119.437772:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437777:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177119.437778:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437782:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437783:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177119.437785:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177119.437787:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177119.437789:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177119.437790:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306466308 00000100:00000001:2.0:1786177119.437792:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306470144 00000100:00000001:2.0:1786177119.437793:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306466308 00000100:00000001:2.0:1786177119.437794:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306470144 00000100:00000001:2.0:1786177119.437795:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177119.437796:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437798:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177119.437799:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177119/real 1786177119] req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177119.437804:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177119.437805:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177119.437806:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177119.437807:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177119.437808:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177119.437809:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177119.437811:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177119.437815:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437819:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177119.437819:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.437820:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177119.437821:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177119.437824:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437828:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177119.437829:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437833:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177119.437834:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.437835:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177119.437836:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.437837:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177119.437838:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177119.437839:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177119.437841:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177119.437842:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177119.437844:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177119.437851:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577856:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177119.437854:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177119.437855:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437859:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177119.437859:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177119.437861:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177119.437862:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177119.437863:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177119.437864:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577984:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177119.437866:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177119.437867:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177119.437868:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177119.437869:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177119.437871:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177119.437872:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177119.437874:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177119.437875:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177119.437877:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177119.437879:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235577984, portal 4 00000100:00000001:2.0:1786177119.437880:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177119.437881:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487986560 : -110371221565056 : ffff9b9e328b9180) 00000100:00000040:2.0:1786177119.437884:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437887:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177119.437889:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177119.437890:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177119.437891:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235577984, offset 0 00000400:00000200:2.0:1786177119.437893:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177119.437896:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177119.437898:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177119.437899:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177119.437901:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177119.437903:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177119.437905:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177119.437908:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426582:426582:256:4294967295] 192.168.202.132@tcp LPNI seq info [426544:426544:8:4294967295] 00000400:00000200:2.0:1786177119.437912:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177119.437918:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177119.437922:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177119.437925:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177119.437928:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177119.437931:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177119.437935:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177119.437937:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177119.437941:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177119.437942:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177119.437945:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177119.437948:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177119.437950:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177119.437951:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177119.437953:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177119.437955:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437960:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177119.437961:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.437965:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437966:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177119.437968:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177119.437970:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177119.437972:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177119.437973:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306466308 00000100:00000001:2.0:1786177119.437975:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306470144 00000100:00000001:2.0:1786177119.437976:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306466308 00000100:00000001:2.0:1786177119.437977:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306470144 00000100:00000001:2.0:1786177119.437978:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177119.437979:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.437980:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177119.437982:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177119/real 1786177119] req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177119.437987:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177119.437988:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177119.437989:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177119.437990:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177119.437991:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177119.437992:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177119.437994:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177119.438002:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438008:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177119.438009:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438011:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177119.438012:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177119.438015:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438022:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177119.438026:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438032:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177119.438035:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438036:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177119.438038:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438039:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177119.438040:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177119.438042:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177119.438044:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177119.438045:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177119.438048:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177119.438053:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235577984:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177119.438055:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438056:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177119.438058:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328baa00 x1872930235577728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438061:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177119.438062:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177119.438063:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786177119.438065:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177119.438066:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177119.438067:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177119.438068:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177119.438069:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:2.0:1786177119.438071:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177119.438071:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177119.438072:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328baa00. 00000100:00000001:2.0:1786177119.438074:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177119.438075:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438076:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177119.438078:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8a80 x1872930235577856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438099:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177119.438100:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177119.438101:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177119.438102:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177119.438103:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177119.438104:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177119.438105:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177119.438106:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:2.0:1786177119.438107:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177119.438108:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177119.438108:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8a80. 00000100:00000001:2.0:1786177119.438109:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177119.438110:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438111:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177119.438113:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9180 x1872930235577984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438117:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177119.438118:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177119.438119:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177119.438120:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177119.438121:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177119.438122:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177119.438124:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177119.438125:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:2.0:1786177119.438127:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177119.438128:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177119.438129:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9180. 00000100:00000001:2.0:1786177119.438131:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177119.438132:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438134:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438137:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177119.438138:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.438140:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177119.438142:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177119.438143:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177119.438147:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438154:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177119.438155:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177119.438157:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177119.438159:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177119.438161:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177119.438164:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235578112:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177119.438166:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177119.438167:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177119.438168:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177119.438169:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177119.438171:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177119.438173:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177119.438175:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177119.438176:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177119.438178:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177119.438180:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235578112, portal 4 00000100:00000001:2.0:1786177119.438181:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177119.438182:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487996416 : -110371221555200 : ffff9b9e328bb800) 00000100:00000040:2.0:1786177119.438186:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438190:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177119.438191:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177119.438192:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177119.438194:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235578112, offset 0 00000400:00000200:2.0:1786177119.438196:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177119.438198:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177119.438200:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177119.438202:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177119.438203:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177119.438205:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177119.438206:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177119.438209:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426583:426583:256:4294967295] 192.168.202.132@tcp LPNI seq info [426545:426545:8:4294967295] 00000400:00000200:2.0:1786177119.438211:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177119.438215:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177119.438218:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177119.438219:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177119.438222:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177119.438223:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177119.438226:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177119.438227:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177119.438230:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177119.438231:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177119.438232:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177119.438235:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177119.438236:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177119.438237:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177119.438239:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177119.438240:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438245:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177119.438247:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177144 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438250:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.438251:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177119.438253:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177119.438254:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177119.438256:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177119.438257:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306466310 00000100:00000001:2.0:1786177119.438258:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306470144 00000100:00000001:2.0:1786177119.438259:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306466310 00000100:00000001:2.0:1786177119.438261:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306470144 00000100:00000001:2.0:1786177119.438262:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177119.438264:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.438265:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177119.438267:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177119/real 1786177119] req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177119.438271:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177119.438272:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177119.438273:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177119.438274:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177119.438275:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177119.438276:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177119.438277:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177119.438282:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438286:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177119.438286:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438287:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177119.438288:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177119.438290:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438293:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177119.438295:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438298:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177119.438300:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438300:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177119.438301:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438302:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177119.438303:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177119.438304:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177119.438306:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177119.438307:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177119.438310:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177119.438314:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235578112:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177119.438316:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438317:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177119.438318:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb800 x1872930235578112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177144 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177119.438321:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177119.438322:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177119.438323:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177119.438324:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177119.438325:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177119.438326:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177119.438327:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177119.438328:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:2.0:1786177119.438329:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177119.438330:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177119.438331:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb800. 00000100:00000001:2.0:1786177119.438332:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177119.438333:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438334:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177119.438336:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177119.438337:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177119.438338:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177119.438339:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177119.948444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177119.948452:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177119.948455:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177119.948461:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177119.948465:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177119.948469:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177119.948472:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177119.948475:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177119.948479:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177120.332435:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177120.332441:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177120.332443:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177120.332447:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177120.332450:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177120.332453:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177120.332453:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177120.332456:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177120.332458:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177120.332459:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177120.332465:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177120.332467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177120.332472:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177120.332474:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177120.332477:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177120.332481:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177120.332483:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177120.332485:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177120.332488:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177120.332492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177120.332493:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177120.332536:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177120.332539:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177120.332541:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786177120.908469:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786177120.908475:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786177120.908479:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1786177120.908612:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1786177120.908623:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1786177120.908626:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786177120.908631:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786177120.908633:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177120.972458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177120.972465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177120.972466:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177120.972477:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177120.972481:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177120.972484:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177120.972487:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177120.972488:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177120.972490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177121.356470:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177121.356475:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177121.356478:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177121.356480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177121.356483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177121.356484:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177121.356485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177121.356489:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177121.356489:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177121.356496:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177121.356499:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177121.356502:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177121.356505:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177121.356508:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177121.356512:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177121.356533:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177121.356537:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177121.356540:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177121.996470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177121.996475:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177121.996477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177121.996489:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177121.996493:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177121.996496:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177121.996499:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177121.996502:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177121.996504:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177122.380419:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177122.380420:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177122.380424:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177122.380427:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177122.380429:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177122.380430:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177122.380431:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177122.380436:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177122.380437:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177122.380438:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177122.380443:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177122.380447:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177122.380450:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177122.380453:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177122.380456:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177122.380480:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177122.380484:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177122.380488:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177123.020444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177123.020451:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177123.020453:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177123.020459:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177123.020462:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177123.020465:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177123.020467:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177123.020470:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177123.020473:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177123.340443:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306470144, at 4306470208 00000100:00000040:1.0:1786177123.340459:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306470144, at 4306470208 00000100:00000010:0.0:1786177123.340464:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c3c0. 00000100:00000010:1.0:1786177123.340470:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cd80. 00000100:00000001:1.0:1786177123.404423:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177123.404429:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177123.404431:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177123.404432:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177123.404436:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177123.404438:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177123.404442:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177123.404442:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177123.404447:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177123.404448:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177123.404453:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177123.404454:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177123.404456:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177123.404457:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177123.404459:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177123.404461:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177123.404464:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177123.404467:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177123.404467:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177123.404470:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177123.404472:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177123.404474:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177123.404475:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177123.404476:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177123.404477:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177123.404479:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177123.404480:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177123.404528:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177123.404531:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177123.404534:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177124.044458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177124.044466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177124.044469:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177124.044476:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177124.044480:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177124.044485:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.044495:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177124.044498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.044501:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.428438:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177124.428444:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177124.428446:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177124.428447:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177124.428451:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177124.428453:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177124.428456:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177124.428457:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177124.428458:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.428462:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177124.428463:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177124.428467:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177124.428468:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.428471:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.428474:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177124.428556:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177124.428561:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177124.428566:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177124.556483:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177124.556493:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177124.556496:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177124.556500:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177124.556504:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177124.556512:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11799 00000400:00000200:2.0:1786177124.556521:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177124.556528:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177124.556531:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177124.556535:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177124.556539:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177124.556542:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177124.556545:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177124.556549:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177124.556551:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177124.556553:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177124.556556:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177124.556558:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177124.556561:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177124.556564:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177124.556567:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177124.556569:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177124.556571:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177124.556574:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177124.556577:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177124.556580:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177124.556583:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177124.556587:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177124.556590:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177124.556593:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177124.556596:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177124.556606:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786177124.556609:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177124.556612:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177124.556614:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177124.556616:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177124.556618:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177124.556622:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177124.556628:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000001:2.0:1786177124.556633:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177124.556641:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177124.556649:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177124.556653:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177124.556656:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177124.556675:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177124.556678:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177124.556680:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177124.556681:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177124.556683:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177124.556685:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177124.556687:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11799 00000400:00000200:2.0:1786177124.556691:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177124.556694:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177124.556696:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177124.556698:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1786177124.556699:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177124.556703:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177124.556703:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177124.556707:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177124.556708:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1786177124.556710:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177124.556713:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177124.556715:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177124.556716:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1786177124.556717:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177124.556718:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177124.556724:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177124.556730:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177124.556732:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177124.556735:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177124.556738:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177124.556739:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177124.556743:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177124.556744:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177124.556747:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177124.556750:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00100000:1.0:1786177124.556753:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235578240:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177124.556757:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177124.556759:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1786177124.556763:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 02000000:00000001:1.0:1786177124.556764:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177124.556766:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177124.556769:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1786177124.556771:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000010:1.0:1786177124.556775:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae400. 00000100:00000001:2.0:1786177124.556776:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177124.556779:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177124.556780:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177124.556786:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1786177124.556789:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177124.556790:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177124.556793:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:1.0:1786177124.556794:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000008:00100000:2.0:1786177124.556797:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177124.556800:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177124.556801:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000001:2.0:1786177124.556802:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177124.556807:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 00000100:00000200:1.0:1786177124.556809:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235578240, portal 25 00000020:00000040:2.0:1786177124.556810:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000100:00000001:1.0:1786177124.556814:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1786177124.556815:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177124.556816:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620992 : -110371239930624 : ffff9b9e31735500) 02000000:00000001:2.0:1786177124.556817:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177124.556820:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177124.556823:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1786177124.556824:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177124.556826:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177124.556832:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:1.0:1786177124.556835:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1786177124.556837:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177124.556838:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177124.556843:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00100000:2.0:1786177124.556846:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1786177124.556848:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235578240, offset 0 00000400:00000200:1.0:1786177124.556858:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177124.556860:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177124.556863:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177124.556866:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177124.556868:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177124.556878:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:2.0:1786177124.556878:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177124.556882:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177124.556885:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1786177124.556886:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177124.556889:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177124.556890:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1786177124.556891:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:2.0:1786177124.556893:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177124.556896:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1786177124.556898:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11799 00000400:00000200:1.0:1786177124.556902:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177124.556906:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177124.556910:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426584:426584:256:4294967295] 192.168.202.132@tcp LPNI seq info [426546:426546:8:4294967295] 00000400:00000200:1.0:1786177124.556914:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177124.556923:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177124.556928:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00080000:2.0:1786177124.556928:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177124.556933:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:1.0:1786177124.556938:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177124.556938:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177124.556942:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000010:1.0:1786177124.556946:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000100:00000001:2.0:1786177124.556946:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177124.556950:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177124.556951:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000800:00000200:1.0:1786177124.556955:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:2.0:1786177124.556956:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:1.0:1786177124.556960:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000020:00000040:2.0:1786177124.556961:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177124.556965:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177124.556968:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000200:1.1:1786177124.556971:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:2.0:1786177124.556974:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:1.1:1786177124.556976:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1786177124.556977:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177124.556981:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177124.556983:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000100:1.0:1786177124.556986:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000020:00000040:2.0:1786177124.556987:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000010:1.0:1786177124.556989:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000100:00080000:2.0:1786177124.556992:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:1.0:1786177124.556995:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1786177124.556995:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177124.556997:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177124.557000:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1786177124.557003:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1786177124.557005:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177124.557007:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177124.557009:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177124.557010:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000008:00000001:2.0:1786177124.557012:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177124.557013:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:2.0:1786177124.557015:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557018:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:2.0:1786177124.557019:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000020:00000040:2.0:1786177124.557023:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000200:1.0:1786177124.557024:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177124.557026:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177124.557029:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177124.557045:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177124.557048:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177124.557051:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557054:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:2.0:1786177124.557056:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aabc00. 00000100:00000040:1.0:1786177124.557060:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177124.557061:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177124.557070:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177124.557072:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557077:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177124.557081:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177124.557083:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557085:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177124.557087:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177124.557093:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177124.557098:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306475264 00000100:00000001:2.0:1786177124.557101:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177124.557104:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:1.0:1786177124.557105:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62de0. 00000100:00080000:2.0:1786177124.557107:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177124.557110:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177124.557111:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177124.557113:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177124.557114:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177124.557121:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306475264 00000100:00080000:2.0:1786177124.557126:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11799 00000100:00000010:1.0:1786177124.557128:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c625a0. 00000400:00000200:2.0:1786177124.557130:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177124.557133:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177124.557135:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177124.557136:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177124.557138:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177124.557143:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177124.557145:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177124.557145:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177124.557148:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177124.557148:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177124.557149:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177124.557151:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177124.557154:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177124.557156:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177124.557157:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1786177124.557159:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177124.557162:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177124.557163:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177124.557167:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177124.557169:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1786177124.557170:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177124.557174:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00100000:1.0:1786177124.557179:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177124/real 1786177124] req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177124.557180:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177124.557186:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177124.557192:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:1.0:1786177124.557195:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1786177124.557197:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1786177124.557198:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177124.557201:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 02000000:00000001:2.0:1786177124.557203:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177124.557205:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000100:00000001:2.0:1786177124.557208:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:1.0:1786177124.557209:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177124.557213:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000008:00100000:2.0:1786177124.557214:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000200:1.0:1786177124.557218:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1786177124.557219:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177124.557226:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177124.557230:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1786177124.557232:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000020:00000040:2.0:1786177124.557237:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177124.557243:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1786177124.557244:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177124.557246:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1786177124.557247:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557250:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177124.557250:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177124.557253:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1786177124.557253:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177124.557258:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177124.557260:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1786177124.557262:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9000. 00000100:00000001:2.0:1786177124.557267:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557269:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177124.557275:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1786177124.557276:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177124.557297:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177124.557298:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177124.557301:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557303:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.557306:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177124.557307:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.557309:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177124.557310:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177124.557312:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1786177124.557314:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177124.557318:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177124.557322:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177124.557328:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177124.557330:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177124.557335:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177124.557340:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1786177124.557341:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235578240:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177124.557342:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177124.557345:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177124.557347:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177124.557349:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177124.557351:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1786177124.557354:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11799 00000100:00000040:1.0:1786177124.557374:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735500 x1872930235578240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557380:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177124.557382:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177124.557386:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae400. 00000400:00000200:2.0:1786177124.557386:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177124.557392:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1786177124.557393:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000020:00000001:1.0:1786177124.557396:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000100:00080000:2.0:1786177124.557397:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000020:00000040:1.0:1786177124.557400:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000040:2.0:1786177124.557402:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000020:00000001:1.0:1786177124.557404:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:2.0:1786177124.557405:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:1.0:1786177124.557408:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaac00. 00000100:00000040:2.0:1786177124.557409:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177124.557412:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:1.0:1786177124.557413:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:2.0:1786177124.557415:0:191280:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1786177124.557416:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000040:2.0:1786177124.557418:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000010:1.0:1786177124.557420:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735500. 00000020:00000001:2.0:1786177124.557421:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177124.557423:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000040:2.0:1786177124.557424:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:1.0:1786177124.557426:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1786177124.557428:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177124.557430:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1786177124.557431:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177124.557435:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177124.557436:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177124.557438:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1786177124.557438:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177124.557439:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177124.557441:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177124.557442:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:1.0:1786177124.557445:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1786177124.557446:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177124.557448:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177124.557449:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557450:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1786177124.557452:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177124.557456:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1786177124.557457:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177124.557461:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177124.557465:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177124.557468:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177124.557472:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1786177124.557473:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557476:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177124.557477:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177124.557483:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:2.0:1786177124.557484:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 02000000:00000001:1.0:1786177124.557486:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1786177124.557487:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177124.557490:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177124.557491:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1786177124.557495:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235578368:192.168.202.132@tcp:38:kworker.0 02000000:00000001:2.0:1786177124.557495:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557499:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1786177124.557501:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177124.557503:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177124.557506:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177124.557507:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177124.557510:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1786177124.557512:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177124.557518:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000010:1.0:1786177124.557521:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae400. 00000100:00000001:2.0:1786177124.557523:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177124.557524:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177124.557530:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177124.557534:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000100:00100000:2.0:1786177124.557535:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177124.557537:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177124.557542:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235578368, portal 10 00000100:00000001:1.0:1786177124.557547:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177124.557549:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469630848 : -110371239920768 : ffff9b9e31737b80) 00000100:00000001:2.0:1786177124.557555:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177124.557558:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177124.557559:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557567:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177124.557567:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177124.557573:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177124.557577:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177124.557581:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235578368, offset 0 00000400:00000200:1.0:1786177124.557589:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177124.557592:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177124.557593:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177124.557600:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000040:2.0:1786177124.557601:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11809) 00000400:00000200:1.0:1786177124.557603:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177124.557608:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177124.557611:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177124.557613:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177124.557616:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426585:426585:256:4294967295] 192.168.202.132@tcp LPNI seq info [426547:426547:8:4294967295] 00000400:00000200:1.0:1786177124.557619:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177124.557624:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177124.557627:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177124.557630:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177124.557633:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177124.557635:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786177124.557638:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177124.557639:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1786177124.557642:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177124.557644:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177124.557646:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177124.557648:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177124.557650:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177124.557652:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177124.557653:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177124.557656:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557661:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177124.557663:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557668:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557669:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177124.557671:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177124.557674:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177124.557676:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177124.557677:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306471428 00000100:00000001:1.0:1786177124.557678:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306475264 00000100:00000001:1.0:1786177124.557680:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306471428 00000100:00000001:1.0:1786177124.557682:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306475264 00000100:00000001:1.0:1786177124.557684:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177124.557685:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557687:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177124.557689:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177124/real 1786177124] req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177124.557694:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177124.557695:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177124.557696:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177124.557697:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177124.557698:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177124.557699:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177124.557701:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177124.557707:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557711:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177124.557712:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.557713:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177124.557714:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177124.557716:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557720:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177124.557722:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557727:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177124.557728:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.557729:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177124.557730:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.557731:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177124.557732:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177124.557735:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177124.557737:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177124.557739:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177124.557741:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177124.557746:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235578368:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177124.557748:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177124.557750:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557754:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177124.557755:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177124.557756:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177124.557758:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177124.557759:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177124.557760:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235578496:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177124.557762:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177124.557764:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177124.557765:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177124.557766:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177124.557768:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177124.557770:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177124.557772:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177124.557774:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177124.557776:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177124.557777:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235578496, portal 10 00000100:00000001:1.0:1786177124.557779:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177124.557780:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469616512 : -110371239935104 : ffff9b9e31734380) 00000100:00000040:1.0:1786177124.557784:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557793:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177124.557795:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177124.557796:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177124.557798:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235578496, offset 0 00000400:00000200:1.0:1786177124.557800:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177124.557802:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177124.557804:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177124.557806:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177124.557807:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177124.557809:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177124.557810:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177124.557812:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426586:426586:256:4294967295] 192.168.202.132@tcp LPNI seq info [426548:426548:8:4294967295] 00000400:00000200:1.0:1786177124.557815:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177124.557820:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177124.557822:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177124.557824:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177124.557827:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177124.557828:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786177124.557831:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177124.557833:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1786177124.557835:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177124.557837:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177124.557839:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177124.557841:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177124.557843:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177124.557844:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177124.557846:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177124.557848:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557853:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177124.557855:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557863:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557864:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177124.557866:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177124.557868:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177124.557870:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177124.557871:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306471428 00000100:00000001:1.0:1786177124.557873:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306475264 00000100:00000001:1.0:1786177124.557874:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306471428 00000100:00000001:1.0:1786177124.557876:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306475264 00000100:00000001:1.0:1786177124.557877:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177124.557878:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.557879:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177124.557881:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177124/real 1786177124] req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177124.557886:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177124.557886:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177124.557888:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177124.557889:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177124.557890:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177124.557891:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177124.557893:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177124.557898:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557902:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177124.557903:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.557903:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177124.557904:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177124.557906:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557911:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177124.557914:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557918:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177124.557920:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.557921:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177124.557921:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.557922:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177124.557924:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177124.557925:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177124.557927:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177124.557928:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177124.557930:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177124.557935:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235578496:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177124.557937:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177124.557939:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557943:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177124.557944:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177124.557945:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177124.557947:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177124.557948:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177124.557950:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235578624:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177124.557951:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177124.557952:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177124.557954:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177124.557955:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177124.557957:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177124.557958:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177124.557960:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177124.557961:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177124.557963:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177124.557965:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235578624, portal 4 00000100:00000001:1.0:1786177124.557966:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177124.557967:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469624576 : -110371239927040 : ffff9b9e31736300) 00000100:00000040:1.0:1786177124.557971:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.557975:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177124.557977:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177124.557978:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177124.557979:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235578624, offset 0 00000400:00000200:1.0:1786177124.557981:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177124.557983:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177124.557985:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177124.557986:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177124.557988:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177124.557989:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177124.557991:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177124.557993:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426587:426587:256:4294967295] 192.168.202.132@tcp LPNI seq info [426549:426549:8:4294967295] 00000400:00000200:1.0:1786177124.557996:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177124.558000:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177124.558002:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177124.558004:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177124.558007:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177124.558008:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786177124.558010:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177124.558012:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1786177124.558015:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177124.558016:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177124.558018:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177124.558020:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177124.558022:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177124.558023:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177124.558024:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177124.558026:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558031:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177124.558049:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558054:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.558055:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177124.558058:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177124.558060:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177124.558062:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177124.558064:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306471428 00000100:00000001:1.0:1786177124.558065:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306475264 00000100:00000001:1.0:1786177124.558067:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306471428 00000100:00000001:1.0:1786177124.558068:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306475264 00000100:00000001:1.0:1786177124.558069:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177124.558071:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.558072:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177124.558074:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177124/real 1786177124] req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177124.558079:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177124.558080:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177124.558081:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177124.558083:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177124.558084:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177124.558086:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177124.558088:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177124.558093:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558097:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177124.558098:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558099:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177124.558100:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177124.558102:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558106:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177124.558108:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558112:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177124.558114:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558115:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177124.558116:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558117:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177124.558118:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177124.558120:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177124.558124:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177124.558126:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177124.558130:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177124.558138:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235578624:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177124.558142:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558144:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177124.558148:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737b80 x1872930235578368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558155:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177124.558157:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177124.558159:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177124.558162:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177124.558163:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177124.558165:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177124.558167:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177124.558169:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:1.0:1786177124.558171:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177124.558173:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177124.558174:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737b80. 00000100:00000001:1.0:1786177124.558176:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177124.558177:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558179:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177124.558181:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734380 x1872930235578496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558186:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177124.558186:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177124.558188:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177124.558189:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177124.558189:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177124.558190:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177124.558191:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177124.558192:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:1.0:1786177124.558193:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177124.558194:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177124.558195:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734380. 00000100:00000001:1.0:1786177124.558196:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177124.558197:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558198:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177124.558200:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736300 x1872930235578624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558203:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177124.558205:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177124.558206:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177124.558207:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177124.558208:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177124.558209:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177124.558210:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177124.558211:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:1.0:1786177124.558212:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177124.558212:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177124.558213:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736300. 00000100:00000001:1.0:1786177124.558215:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177124.558215:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558217:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558219:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177124.558220:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.558221:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177124.558222:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177124.558223:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177124.558225:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558229:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177124.558230:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177124.558231:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177124.558232:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177124.558234:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177124.558235:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235578752:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177124.558238:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177124.558239:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177124.558240:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177124.558241:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177124.558243:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177124.558244:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177124.558247:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177124.558249:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177124.558251:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177124.558253:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235578752, portal 4 00000100:00000001:1.0:1786177124.558255:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177124.558256:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469615616 : -110371239936000 : ffff9b9e31734000) 00000100:00000040:1.0:1786177124.558259:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558264:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177124.558265:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177124.558267:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177124.558268:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235578752, offset 0 00000400:00000200:1.0:1786177124.558271:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177124.558273:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177124.558275:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177124.558277:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177124.558279:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177124.558281:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177124.558282:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177124.558284:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426588:426588:256:4294967295] 192.168.202.132@tcp LPNI seq info [426550:426550:8:4294967295] 00000400:00000200:1.0:1786177124.558287:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177124.558291:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177124.558294:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177124.558296:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177124.558298:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177124.558300:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1786177124.558303:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177124.558305:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1786177124.558307:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177124.558309:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177124.558311:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177124.558313:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177124.558315:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177124.558316:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177124.558318:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177124.558320:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558329:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177124.558331:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177149 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558335:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.558336:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177124.558338:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177124.558340:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177124.558342:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177124.558343:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306471430 00000100:00000001:1.0:1786177124.558345:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306475264 00000100:00000001:1.0:1786177124.558346:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306471430 00000100:00000001:1.0:1786177124.558347:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306475264 00000100:00000001:1.0:1786177124.558349:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177124.558350:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.558351:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177124.558353:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177124/real 1786177124] req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177124.558373:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177124.558374:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177124.558375:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177124.558376:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177124.558377:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177124.558379:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177124.558381:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177124.558386:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558391:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177124.558392:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558393:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177124.558395:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177124.558397:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558401:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177124.558404:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558408:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177124.558409:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558410:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177124.558411:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558412:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177124.558413:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177124.558415:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177124.558416:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177124.558418:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177124.558420:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177124.558425:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235578752:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177124.558427:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558428:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177124.558430:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734000 x1872930235578752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177149 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177124.558435:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177124.558436:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177124.558437:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177124.558438:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177124.558439:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177124.558440:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177124.558442:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177124.558443:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:1.0:1786177124.558444:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177124.558445:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177124.558446:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734000. 00000100:00000001:1.0:1786177124.558448:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177124.558449:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558450:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177124.558452:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177124.558453:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177124.558455:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177124.558456:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177125.068467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177125.068475:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177125.068478:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177125.068484:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177125.068488:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177125.068493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177125.068496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177125.068499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177125.068502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177125.452406:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177125.452412:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177125.452414:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177125.452419:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177125.452422:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177125.452426:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177125.452428:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177125.452430:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177125.452433:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177125.452437:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177125.452442:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177125.452447:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177125.452449:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177125.452452:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177125.452454:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177125.452520:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177125.452523:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177125.452526:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177126.092453:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177126.092460:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177126.092462:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177126.092467:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177126.092471:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177126.092474:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177126.092476:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177126.092479:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177126.092481:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177126.476474:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177126.476480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177126.476483:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177126.476486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177126.476488:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177126.476496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177126.476496:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177126.476500:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177126.476503:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177126.476507:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177126.476510:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177126.476512:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177126.476517:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177126.476522:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177126.476526:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177126.476576:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177126.476580:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177126.476583:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177127.116506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177127.116517:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177127.116523:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177127.116530:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177127.116536:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177127.116540:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177127.116543:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177127.116546:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177127.116549:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177127.500469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177127.500475:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177127.500476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177127.500479:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177127.500481:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177127.500487:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177127.500488:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177127.500492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177127.500494:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177127.500498:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177127.500499:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177127.500501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177127.500504:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177127.500509:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177127.500513:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786177127.500564:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786177127.500567:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786177127.500570:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177128.140433:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177128.140441:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177128.140443:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177128.140448:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177128.140452:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177128.140455:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177128.140457:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177128.140460:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177128.140463:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177128.460465:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306475264, at 4306475328 00000100:00000040:1.0:1786177128.460465:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306475264, at 4306475328 00000100:00000010:1.0:1786177128.460480:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c625a0. 00000100:00000010:0.0:1786177128.460483:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62de0. 00000100:00000001:1.0:1786177128.524477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177128.524482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177128.524483:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177128.524486:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177128.524488:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177128.524491:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177128.524495:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177128.524496:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177128.524501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177128.524501:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177128.524508:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177128.524510:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177128.524511:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177128.524515:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177128.524516:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177128.524576:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177128.524580:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177128.524583:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177129.164629:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177129.164635:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177129.164637:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177129.164641:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177129.164645:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177129.164649:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177129.164652:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177129.164655:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177129.164658:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.548461:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177129.548467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177129.548468:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177129.548472:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.548473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177129.548479:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177129.548480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177129.548485:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177129.548486:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177129.548489:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177129.548489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177129.548492:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.548494:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177129.548497:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177129.548497:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177129.548500:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.548501:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177129.548504:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177129.548507:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177129.548510:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177129.548512:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786177129.548536:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786177129.548539:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786177129.548542:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177129.676494:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177129.676506:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177129.676509:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177129.676513:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177129.676518:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177129.676525:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11804 00000400:00000200:1.0:1786177129.676534:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177129.676541:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177129.676544:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177129.676546:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676549:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177129.676551:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676552:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177129.676556:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177129.676557:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177129.676559:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177129.676562:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177129.676564:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177129.676566:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177129.676568:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676569:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177129.676570:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676572:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177129.676574:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177129.676577:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676578:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177129.676580:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177129.676583:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177129.676585:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.676587:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177129.676589:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177129.676597:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9f80. 00000020:00000040:1.0:1786177129.676600:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177129.676603:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177129.676604:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.676607:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177129.676609:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177129.676612:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177129.676618:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae400. 00000100:00000001:1.0:1786177129.676624:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177129.676631:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177129.676639:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177129.676643:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177129.676646:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4f3e5a2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177129.676668:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177129.676671:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177129.676674:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177129.676675:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177129.676677:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177129.676681:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177129.676683:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11804 00000400:00000200:1.0:1786177129.676688:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177129.676692:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177129.676695:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177129.676698:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676701:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177129.676701:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1786177129.676707:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177129.676707:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177129.676711:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177129.676713:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177129.676719:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000040:2.0:1786177129.676720:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177129.676721:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177129.676726:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177129.676730:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177129.676733:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177129.676734:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177129.676737:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676739:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177129.676742:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1786177129.676743:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177129.676747:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.676748:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:2.0:1786177129.676753:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177129.676755:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1786177129.676758:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 02000000:00000001:2.0:1786177129.676759:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177129.676763:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00100000:2.0:1786177129.676767:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235578880:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177129.676768:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177129.676772:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177129.676775:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177129.676778:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177129.676780:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177129.676782:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177129.676784:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.676785:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:2.0:1786177129.676788:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1786177129.676790:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177129.676793:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177129.676794:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:1.0:1786177129.676795:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177129.676798:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177129.676801:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb480. 00000020:00000040:1.0:1786177129.676804:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177129.676805:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177129.676806:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.676808:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177129.676809:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1786177129.676811:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177129.676812:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177129.676817:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 02000000:00000010:1.0:1786177129.676819:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac800. 00000100:00000001:1.0:1786177129.676823:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177129.676824:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00100000:1.0:1786177129.676829:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1786177129.676832:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235578880, portal 25 00000100:00000001:2.0:1786177129.676838:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177129.676840:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487990144 : -110371221561472 : ffff9b9e328b9f80) 00000100:00000001:1.0:1786177129.676849:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1786177129.676849:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177129.676853:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177129.676860:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b192a357] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.676868:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1786177129.676869:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177129.676873:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1786177129.676875:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00080000:1.0:1786177129.676877:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177129.676879:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:2.0:1786177129.676879:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00080000:1.0:1786177129.676883:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1786177129.676885:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235578880, offset 0 00000100:00000001:1.0:1786177129.676886:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177129.676891:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11804 00000400:00000200:1.0:1786177129.676896:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177129.676901:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:2.0:1786177129.676901:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:1.0:1786177129.676905:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177129.676907:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676908:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177129.676910:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676913:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177129.676914:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177129.676915:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177129.676916:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177129.676918:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177129.676919:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177129.676924:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786177129.676924:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177129.676929:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676931:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177129.676932:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676934:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177129.676937:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:2.0:1786177129.676938:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00080000:1.0:1786177129.676942:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177129.676946:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177129.676947:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1786177129.676948:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1786177129.676950:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177129.676953:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177129.676955:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:1.0:1786177129.676956:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177129.676959:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1786177129.676962:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000008:00000001:1.0:1786177129.676963:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.676967:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177129.676968:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000010:1.0:1786177129.676971:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb100. 00000400:00000200:2.0:1786177129.676976:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426589:426589:256:4294967295] 192.168.202.132@tcp LPNI seq info [426551:426551:8:4294967295] 00000020:00000040:1.0:1786177129.676977:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177129.676981:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1786177129.676996:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177129.676999:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.677002:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177129.677004:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177129.677007:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177129.677011:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 02000000:00000010:1.0:1786177129.677013:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad000. 00000100:00000001:1.0:1786177129.677018:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177129.677022:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00100000:1.0:1786177129.677026:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:2.0:1786177129.677028:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00000001:1.0:1786177129.677036:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177129.677039:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177129.677040:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:1.0:1786177129.677045:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f57195c8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1786177129.677047:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:1.0:1786177129.677059:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786177129.677060:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:1.0:1786177129.677062:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177129.677069:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177129.677071:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.1:1786177129.677071:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00080000:1.0:1786177129.677073:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177129.677077:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177129.677082:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11804 00000800:00000100:2.0:1786177129.677085:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177129.677088:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000010:2.0:1786177129.677091:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:1.0:1786177129.677092:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177129.677095:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177129.677099:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.677102:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177129.677102:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:1.0:1786177129.677105:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.677109:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177129.677112:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177129.677115:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000400:00000200:2.0:1786177129.677115:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000020:00000001:1.0:1786177129.677118:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177129.677121:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000400:00000200:2.0:1786177129.677122:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000020:00000001:1.0:1786177129.677123:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177129.677128:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000010:2.0:1786177129.677128:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000040:1.0:1786177129.677132:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.677135:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177129.677136:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1786177129.677138:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.677141:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000200:2.0:1786177129.677144:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177129.677145:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:1.0:1786177129.677154:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177129.677161:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677161:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1786177129.677165:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1786177129.677167:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177129.677170:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177129.677177:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677178:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.677182:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177129.677183:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:1.0:1786177129.677188:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1786177129.677190:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1786177129.677194:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177129.677198:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177129.677200:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677203:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306480384 00000100:00000010:1.0:1786177129.677207:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328ba680. 00000100:00000010:2.0:1786177129.677210:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c720. 00000020:00000040:1.0:1786177129.677212:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177129.677217:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177129.677218:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1786177129.677220:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.677224:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177129.677226:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1786177129.677227:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306480384 02000000:00000001:1.0:1786177129.677229:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177129.677231:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c7e0. 00000100:00000001:2.0:1786177129.677235:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 02000000:00000010:1.0:1786177129.677236:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac000. 00000100:00000001:2.0:1786177129.677238:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177129.677241:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677241:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677247:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177129.677250:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1786177129.677253:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177129/real 1786177129] req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177129.677272:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177129.677276:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1786177129.677276:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177129.677279:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:1.0:1786177129.677282:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000497a117c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1786177129.677283:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177129.677287:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177129.677291:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177129.677296:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1786177129.677297:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177129.677300:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177129.677302:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177129.677303:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177129.677304:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177129.677306:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177129.677311:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11804 00000100:00000200:2.0:1786177129.677312:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177129.677317:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177129.677324:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177129.677328:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000200:2.0:1786177129.677332:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177129.677333:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.677338:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177129.677342:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.677346:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177129.677351:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177129.677353:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:1.0:1786177129.677354:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177129.677383:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1786177129.677383:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177129.677385:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:2.0:1786177129.677388:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:1.0:1786177129.677389:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177129.677390:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1786177129.677394:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00100000:2.0:1786177129.677398:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177129.677401:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.677404:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177129.677406:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177129.677411:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177129.677414:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177129.677417:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000040:2.0:1786177129.677422:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177129.677425:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177129.677433:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677437:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1786177129.677439:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177129.677441:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177129.677445:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177129.677445:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177129.677448:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177129.677451:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677452:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177129.677455:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177129.677456:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00080000:2.0:1786177129.677459:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1786177129.677461:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177129.677466:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.677468:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177129.677469:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000010:1.0:1786177129.677472:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9c00. 00000100:00000001:2.0:1786177129.677474:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:1.0:1786177129.677477:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000040:2.0:1786177129.677480:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177129.677482:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177129.677485:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177129.677490:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177129.677493:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1786177129.677496:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235578880:192.168.202.132@tcp:250:kworker.0 02000000:00000001:1.0:1786177129.677497:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677502:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1786177129.677505:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261adc00. 00000100:00000001:2.0:1786177129.677505:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177129.677510:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9f80 x1872930235578880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177129.677513:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677520:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177129.677524:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:1.0:1786177129.677525:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1786177129.677527:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177129.677532:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177129.677535:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177129.677538:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000001:1.0:1786177129.677542:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177129.677545:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177129.677547:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177129.677550:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae400. 00000100:00000040:1.0:1786177129.677551:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3ea07ed] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177129.677556:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177129.677559:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177129.677563:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9f80. 00000100:00000001:1.0:1786177129.677564:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677568:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000040:1.0:1786177129.677570:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11814) 00000100:00000001:2.0:1786177129.677571:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.677574:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.677580:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177129.677582:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677584:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177129.677586:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177129.677588:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177129.677591:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.677597:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177129.677598:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177129.677599:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177129.677600:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177129.677602:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177129.677604:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579008:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177129.677606:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177129.677608:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177129.677609:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177129.677610:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177129.677613:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177129.677614:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177129.677617:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177129.677619:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177129.677621:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177129.677624:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235579008, portal 10 00000100:00000001:2.0:1786177129.677627:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177129.677628:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487995520 : -110371221556096 : ffff9b9e328bb480) 00000100:00000040:2.0:1786177129.677635:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.677642:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177129.677644:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177129.677647:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177129.677649:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235579008, offset 0 00000400:00000200:2.0:1786177129.677653:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177129.677658:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177129.677663:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177129.677666:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177129.677669:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177129.677672:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177129.677674:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177129.677678:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426590:426590:256:4294967295] 192.168.202.132@tcp LPNI seq info [426552:426552:8:4294967295] 00000400:00000200:2.0:1786177129.677682:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177129.677688:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177129.677691:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177129.677693:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177129.677696:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177129.677698:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177129.677702:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177129.677704:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177129.677707:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177129.677708:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177129.677710:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177129.677713:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177129.677715:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177129.677717:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177129.677718:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177129.677721:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.677727:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177129.677729:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.677733:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677734:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177129.677737:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177129.677739:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177129.677741:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177129.677743:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306476548 00000100:00000001:2.0:1786177129.677745:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306480384 00000100:00000001:2.0:1786177129.677747:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306476548 00000100:00000001:2.0:1786177129.677748:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306480384 00000100:00000001:2.0:1786177129.677750:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177129.677751:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677752:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177129.677755:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177129/real 1786177129] req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177129.677759:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177129.677760:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177129.677762:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177129.677763:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177129.677764:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177129.677766:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177129.677768:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177129.677773:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.677777:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177129.677777:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.677779:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177129.677779:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177129.677782:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.677786:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177129.677788:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.677793:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177129.677795:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.677796:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177129.677797:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.677799:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177129.677801:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177129.677802:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177129.677804:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177129.677806:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177129.677808:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177129.677813:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579008:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177129.677815:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177129.677817:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.677821:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177129.677823:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177129.677824:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177129.677825:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177129.677827:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177129.677828:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579136:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177129.677830:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177129.677831:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177129.677832:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177129.677833:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177129.677836:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177129.677837:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177129.677839:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177129.677841:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177129.677843:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177129.677845:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235579136, portal 10 00000100:00000001:2.0:1786177129.677846:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177129.677847:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487994624 : -110371221556992 : ffff9b9e328bb100) 00000100:00000040:2.0:1786177129.677851:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.677855:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177129.677857:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177129.677858:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177129.677860:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235579136, offset 0 00000400:00000200:2.0:1786177129.677862:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177129.677865:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177129.677867:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177129.677869:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177129.677871:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177129.677873:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177129.677874:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177129.677876:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426591:426591:256:4294967295] 192.168.202.132@tcp LPNI seq info [426553:426553:8:4294967295] 00000400:00000200:2.0:1786177129.677879:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177129.677883:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177129.677886:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177129.677889:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177129.677893:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177129.677896:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177129.677900:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177129.677902:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177129.677906:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177129.677908:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177129.677911:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177129.677915:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177129.677918:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177129.677920:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177129.677923:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177129.677927:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.677942:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177129.677945:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.677949:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677950:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177129.677953:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177129.677955:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177129.677958:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177129.677959:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306476548 00000100:00000001:2.0:1786177129.677961:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306480384 00000100:00000001:2.0:1786177129.677963:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306476548 00000100:00000001:2.0:1786177129.677964:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306480384 00000100:00000001:2.0:1786177129.677966:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177129.677967:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.677968:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177129.677970:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177129/real 1786177129] req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177129.677976:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177129.677976:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177129.677978:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177129.677979:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177129.677980:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177129.677982:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177129.677997:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177129.678004:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678008:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177129.678009:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678011:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177129.678012:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177129.678014:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678018:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177129.678021:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678025:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177129.678027:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678028:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177129.678029:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678030:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177129.678031:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177129.678033:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177129.678035:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177129.678036:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177129.678038:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177129.678044:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579136:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177129.678046:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177129.678048:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678052:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177129.678053:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177129.678055:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177129.678056:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177129.678057:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177129.678061:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579264:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177129.678064:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177129.678066:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177129.678068:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177129.678069:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177129.678073:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177129.678075:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177129.678079:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177129.678081:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177129.678085:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177129.678088:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235579264, portal 4 00000100:00000001:2.0:1786177129.678090:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177129.678092:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487991936 : -110371221559680 : ffff9b9e328ba680) 00000100:00000040:2.0:1786177129.678099:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678106:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177129.678109:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177129.678111:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177129.678114:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235579264, offset 0 00000400:00000200:2.0:1786177129.678117:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177129.678120:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177129.678122:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177129.678124:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177129.678126:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177129.678128:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177129.678129:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177129.678132:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426592:426592:256:4294967295] 192.168.202.132@tcp LPNI seq info [426554:426554:8:4294967295] 00000400:00000200:2.0:1786177129.678134:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177129.678139:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177129.678142:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177129.678144:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177129.678147:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177129.678149:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177129.678152:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177129.678154:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177129.678157:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177129.678158:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177129.678160:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177129.678163:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177129.678165:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177129.678166:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177129.678167:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177129.678170:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678176:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177129.678178:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678182:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.678183:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177129.678185:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177129.678187:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177129.678189:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177129.678190:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306476548 00000100:00000001:2.0:1786177129.678192:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306480384 00000100:00000001:2.0:1786177129.678194:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306476548 00000100:00000001:2.0:1786177129.678195:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306480384 00000100:00000001:2.0:1786177129.678197:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177129.678198:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.678199:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177129.678202:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177129/real 1786177129] req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177129.678206:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177129.678208:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177129.678209:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177129.678210:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177129.678211:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177129.678213:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177129.678215:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177129.678219:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678223:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177129.678224:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678225:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177129.678226:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177129.678228:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678232:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177129.678234:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678239:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177129.678240:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678242:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177129.678242:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678244:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177129.678245:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177129.678246:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177129.678248:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177129.678249:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177129.678252:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177129.678260:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579264:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177129.678263:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177129.678265:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678269:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177129.678270:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177129.678271:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177129.678272:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177129.678273:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177129.678275:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579392:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177129.678277:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177129.678278:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177129.678279:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177129.678280:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177129.678282:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177129.678283:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177129.678286:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177129.678287:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177129.678289:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177129.678291:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235579392, portal 4 00000100:00000001:2.0:1786177129.678292:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177129.678293:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487989248 : -110371221562368 : ffff9b9e328b9c00) 00000100:00000040:2.0:1786177129.678297:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678301:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177129.678302:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177129.678304:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177129.678305:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235579392, offset 0 00000400:00000200:2.0:1786177129.678307:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177129.678310:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177129.678312:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177129.678314:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177129.678315:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177129.678317:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177129.678318:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177129.678320:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426593:426593:256:4294967295] 192.168.202.132@tcp LPNI seq info [426555:426555:8:4294967295] 00000400:00000200:2.0:1786177129.678323:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177129.678327:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177129.678329:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177129.678331:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177129.678333:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177129.678335:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177129.678338:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177129.678339:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177129.678342:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177129.678343:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177129.678345:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177129.678347:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177129.678349:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177129.678350:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177129.678352:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177129.678354:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678375:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177129.678377:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177154 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678382:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.678383:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177129.678385:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177129.678387:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177129.678389:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177129.678391:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306476551 00000100:00000001:2.0:1786177129.678392:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306480384 00000100:00000001:2.0:1786177129.678394:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306476551 00000100:00000001:2.0:1786177129.678396:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306480384 00000100:00000001:2.0:1786177129.678397:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177129.678398:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.678399:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177129.678401:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177129/real 1786177129] req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177129.678406:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177129.678407:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177129.678408:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177129.678410:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177129.678411:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177129.678412:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177129.678414:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177129.678420:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678423:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177129.678424:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678425:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177129.678426:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177129.678428:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678432:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177129.678434:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678438:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177129.678440:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678441:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177129.678442:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678443:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177129.678444:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177129.678446:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177129.678448:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177129.678449:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177129.678451:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177129.678456:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579392:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177129.678458:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678459:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177129.678461:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb480 x1872930235579008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678465:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177129.678466:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177129.678467:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177129.678469:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177129.678470:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177129.678471:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177129.678473:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177129.678474:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:2.0:1786177129.678476:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177129.678477:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177129.678478:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb480. 00000100:00000001:2.0:1786177129.678480:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177129.678481:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678482:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177129.678484:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb100 x1872930235579136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678493:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177129.678494:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177129.678495:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177129.678496:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177129.678497:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177129.678498:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177129.678499:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177129.678500:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:2.0:1786177129.678501:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177129.678502:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177129.678503:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb100. 00000100:00000001:2.0:1786177129.678504:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177129.678505:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678506:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177129.678507:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328ba680 x1872930235579264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678511:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177129.678512:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177129.678513:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177129.678514:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177129.678515:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177129.678515:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177129.678516:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177129.678517:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:2.0:1786177129.678518:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177129.678519:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177129.678520:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328ba680. 00000100:00000001:2.0:1786177129.678521:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177129.678522:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678523:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177129.678524:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9c00 x1872930235579392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177154 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177129.678528:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177129.678529:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177129.678530:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177129.678532:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177129.678532:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177129.678533:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177129.678534:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177129.678536:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261adc00. 02000000:00000001:2.0:1786177129.678537:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177129.678537:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177129.678538:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9c00. 00000100:00000001:2.0:1786177129.678539:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177129.678540:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678541:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177129.678544:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177129.678545:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177129.678546:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177129.678547:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177130.188451:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177130.188458:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177130.188460:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177130.188466:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177130.188471:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177130.188475:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177130.188478:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177130.188481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177130.188484:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177130.572466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177130.572473:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177130.572478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177130.572479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177130.572482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177130.572484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177130.572488:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177130.572494:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177130.572498:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177130.572502:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177130.572508:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177130.572513:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177130.572517:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177130.572520:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177130.572523:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177130.572555:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177130.572559:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177130.572564:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177131.212611:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177131.212618:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177131.212620:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177131.212625:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177131.212630:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177131.212642:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177131.212645:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177131.212648:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177131.212654:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177131.596460:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177131.596461:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177131.596466:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177131.596468:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177131.596471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177131.596472:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177131.596475:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177131.596477:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177131.596479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177131.596484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177131.596484:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177131.596489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177131.596492:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177131.596494:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177131.596497:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786177131.596506:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786177131.596509:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786177131.596512:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177132.236480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177132.236489:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177132.236492:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177132.236498:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177132.236502:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177132.236506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177132.236509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177132.236512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177132.236515:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177132.620461:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177132.620467:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177132.620469:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177132.620474:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177132.620477:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177132.620480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177132.620482:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177132.620484:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177132.620486:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177132.620580:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177132.620584:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177132.620586:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177132.620588:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177132.620591:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177132.620593:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177132.620594:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177132.620596:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177132.620599:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177132.620600:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177132.620602:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177132.620603:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177132.620604:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177132.620605:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177132.620606:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177132.620607:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177132.620608:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177132.620609:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177132.620666:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177132.620670:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177132.620673:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177133.260464:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177133.260471:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177133.260473:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177133.260478:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177133.260482:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177133.260485:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177133.260487:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177133.260489:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177133.260492:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177133.580467:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306480384, at 4306480448 00000100:00000040:1.0:1786177133.580467:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306480384, at 4306480448 00000100:00000010:0.0:1786177133.580483:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c720. 00000100:00000010:1.0:1786177133.580486:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c7e0. 00000100:00000001:1.0:1786177133.644449:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177133.644453:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177133.644458:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177133.644459:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177133.644463:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177133.644467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177133.644469:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177133.644471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177133.644476:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177133.644478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177133.644481:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177133.644482:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177133.644486:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177133.644489:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177133.644491:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786177133.644509:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786177133.644514:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786177133.644517:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177134.284498:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177134.284505:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177134.284508:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177134.284514:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177134.284520:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177134.284524:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177134.284527:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177134.284531:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177134.284535:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.668503:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177134.668506:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177134.668511:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.668513:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177134.668517:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177134.668518:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177134.668520:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177134.668525:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177134.668526:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177134.668529:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.668532:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177134.668537:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.668540:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177134.668543:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.668548:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177134.668587:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177134.668592:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177134.668596:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177134.796506:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177134.796518:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177134.796520:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177134.796524:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177134.796529:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177134.796537:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11809 00000400:00000200:2.0:1786177134.796545:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177134.796552:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177134.796555:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177134.796558:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177134.796561:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177134.796564:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177134.796566:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177134.796571:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177134.796573:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177134.796576:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177134.796579:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177134.796581:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177134.796585:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177134.796588:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177134.796590:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177134.796592:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177134.796595:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177134.796598:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177134.796601:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177134.796604:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177134.796607:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177134.796610:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177134.796613:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.796616:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177134.796618:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177134.796628:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 00000020:00000040:2.0:1786177134.796632:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177134.796636:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177134.796638:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.796642:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177134.796645:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177134.796648:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177134.796656:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786177134.796662:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177134.796671:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177134.796682:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177134.796686:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177134.796690:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177134.796708:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177134.796711:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177134.796713:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177134.796715:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177134.796716:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177134.796718:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177134.796720:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11809 00000400:00000200:2.0:1786177134.796723:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177134.796725:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177134.796727:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177134.796729:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177134.796731:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177134.796733:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177134.796735:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177134.796736:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177134.796737:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177134.796738:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177134.796740:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177134.796741:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177134.796743:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177134.796745:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177134.796746:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177134.796747:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177134.796751:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177134.796753:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177134.796754:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177134.796758:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:1.0:1786177134.796760:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1786177134.796761:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177134.796765:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177134.796766:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177134.796771:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1786177134.796772:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177134.796773:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.796776:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177134.796780:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177134.796785:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.796789:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.796793:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000010:2.0:1786177134.796794:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000100:00000001:1.0:1786177134.796798:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177134.796799:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177134.796804:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177134.796806:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177134.796806:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.796810:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177134.796812:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177134.796813:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177134.796815:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177134.796820:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177134.796821:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177134.796827:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177134.796828:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235579520:192.168.202.132@tcp:250:kworker.0 00000100:00100000:2.0:1786177134.796833:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177134.796837:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177134.796842:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1786177134.796843:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177134.796846:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177134.796848:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177134.796850:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177134.796853:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1786177134.796859:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177134.796862:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae000. 00000100:00080000:2.0:1786177134.796863:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1786177134.796868:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177134.796869:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177134.796871:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177134.796874:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177134.796876:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177134.796879:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:2.0:1786177134.796880:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11809 00000400:00000200:2.0:1786177134.796885:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177134.796887:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000200:2.0:1786177134.796888:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177134.796892:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177134.796895:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177134.796896:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177134.796898:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177134.796899:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000001:2.0:1786177134.796900:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177134.796905:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177134.796906:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177134.796907:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177134.796908:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000100:00000200:1.0:1786177134.796910:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235579520, portal 25 00000020:00000001:2.0:1786177134.796910:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177134.796912:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000001:1.0:1786177134.796917:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1786177134.796918:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177134.796922:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177134.796923:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469626368 : -110371239925248 : ffff9b9e31736a00) 00000100:00000040:2.0:1786177134.796924:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177134.796929:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177134.796931:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177134.796948:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000040:1.0:1786177134.796951:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177134.796951:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177134.796955:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177134.796958:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177134.796962:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.796966:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177134.796968:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1786177134.796971:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:2.0:1786177134.796972:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.796975:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177134.796979:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000010:2.0:1786177134.796980:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000020:00000040:2.0:1786177134.796983:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000010:1.0:1786177134.796985:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 02000000:00000001:2.0:1786177134.796986:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177134.796988:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.796991:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1786177134.796993:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235579520, offset 0 02000000:00000001:2.0:1786177134.796993:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177134.796996:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177134.797002:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000400:00000200:1.0:1786177134.797006:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177134.797006:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177134.797012:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177134.797019:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177134.797021:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177134.797024:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177134.797030:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177134.797036:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177134.797041:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177134.797045:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177134.797048:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177134.797052:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426594:426594:256:4294967295] 192.168.202.132@tcp LPNI seq info [426556:426556:8:4294967295] 00000400:00000200:1.0:1786177134.797057:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177134.797064:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177134.797077:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177134.797079:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177134.797083:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177134.797086:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177134.797090:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.0:1786177134.797091:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00080000:2.0:1786177134.797092:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177134.797098:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177134.797100:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11809 00000800:00000010:1.0:1786177134.797101:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000400:00000200:2.0:1786177134.797106:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177134.797114:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000200:1.0:1786177134.797115:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:2.0:1786177134.797119:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1786177134.797122:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000040:2.0:1786177134.797125:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177134.797130:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1786177134.797134:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:2.0:1786177134.797136:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:1.1:1786177134.797140:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1786177134.797142:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000100:1.0:1786177134.797149:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177134.797153:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000020:00000001:2.0:1786177134.797155:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1786177134.797162:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:2.0:1786177134.797162:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177134.797166:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177134.797169:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000400:00000200:1.0:1786177134.797172:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000020:00000001:2.0:1786177134.797172:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177134.797177:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000200:1.0:1786177134.797180:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000100:00000040:2.0:1786177134.797181:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177134.797186:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:2.0:1786177134.797186:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177134.797190:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177134.797192:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177134.797193:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:2.0:1786177134.797194:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177134.797198:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177134.797202:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177134.797203:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177134.797204:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177134.797208:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177134.797212:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.797217:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177134.797219:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177134.797223:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.797224:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1786177134.797225:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177134.797230:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000100:00000040:1.0:1786177134.797232:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177134.797233:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177134.797237:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177134.797239:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.797243:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177134.797246:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177134.797249:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.797252:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177134.797253:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:1.0:1786177134.797257:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177134.797258:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177134.797264:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177134.797266:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177134.797275:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177134.797278:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306485504 00000100:00000001:2.0:1786177134.797279:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177134.797283:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177134.797287:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c625a0. 00000100:00000040:2.0:1786177134.797288:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177134.797297:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.797298:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1786177134.797299:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177134.797305:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177134.797306:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177134.797307:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177134.797310:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306485504 00000100:00000001:2.0:1786177134.797310:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177134.797312:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11809 00000100:00000010:1.0:1786177134.797317:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c626c0. 00000400:00000200:2.0:1786177134.797318:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177134.797324:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177134.797325:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1786177134.797326:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177134.797332:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177134.797333:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177134.797334:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177134.797337:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177134.797340:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.797340:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177134.797346:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177134.797347:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000020:00000040:2.0:1786177134.797347:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177134.797351:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177134.797354:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00100000:1.0:1786177134.797381:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177134/real 1786177134] req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177134.797398:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000001:1.0:1786177134.797403:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1786177134.797403:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000001:1.0:1786177134.797407:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:2.0:1786177134.797409:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177134.797412:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000100:00000001:2.0:1786177134.797413:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177134.797417:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177134.797420:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000100:00000001:2.0:1786177134.797420:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:1.0:1786177134.797426:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000020:00000040:2.0:1786177134.797427:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177134.797432:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177134.797433:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1786177134.797435:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177134.797438:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177134.797440:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1786177134.797442:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177134.797442:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.797446:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177134.797449:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177134.797454:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.797458:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177134.797463:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000100:00000200:1.0:1786177134.797467:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177134.797467:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177134.797470:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177134.797473:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177134.797476:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177134.797479:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177134.797481:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.797488:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:2.0:1786177134.797488:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:1.0:1786177134.797493:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177134.797493:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.797499:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:2.0:1786177134.797501:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177134.797503:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177134.797512:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1786177134.797514:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177134.797515:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177134.797522:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177134.797533:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177134.797537:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11819) 00000100:00000001:1.0:1786177134.797545:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177134.797553:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.797563:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177134.797567:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.797569:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177134.797571:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.797573:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177134.797576:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177134.797579:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177134.797584:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177134.797586:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177134.797599:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177134.797610:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235579520:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177134.797615:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.797618:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177134.797621:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736a00 x1872930235579520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.797629:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177134.797632:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177134.797635:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177134.797639:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177134.797641:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177134.797643:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177134.797646:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177134.797648:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:1.0:1786177134.797653:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177134.797655:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177134.797657:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736a00. 00000100:00000001:1.0:1786177134.797661:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177134.797662:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.797665:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.797672:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177134.797674:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.797677:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177134.797678:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177134.797680:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177134.797684:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.797693:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177134.797695:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177134.797698:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177134.797700:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177134.797703:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177134.797707:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579648:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177134.797711:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177134.797714:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177134.797716:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177134.797718:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177134.797721:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177134.797724:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177134.797728:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177134.797732:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177134.797736:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177134.797740:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235579648, portal 10 00000100:00000001:1.0:1786177134.797742:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177134.797743:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469625472 : -110371239926144 : ffff9b9e31736680) 00000100:00000040:1.0:1786177134.797750:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.797758:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177134.797760:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177134.797764:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177134.797766:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235579648, offset 0 00000400:00000200:1.0:1786177134.797771:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177134.797776:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177134.797780:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177134.797784:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177134.797787:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177134.797790:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177134.797793:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177134.797796:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426595:426595:256:4294967295] 192.168.202.132@tcp LPNI seq info [426557:426557:8:4294967295] 00000400:00000200:1.0:1786177134.797801:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177134.797808:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177134.797813:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177134.797817:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177134.797822:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177134.797825:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786177134.797830:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177134.797833:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1786177134.797838:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177134.797840:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177134.797844:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177134.797847:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177134.797851:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177134.797853:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177134.797856:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177134.797860:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.797869:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177134.797873:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.797881:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.797883:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177134.797887:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177134.797891:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177134.797894:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177134.797897:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306481668 00000100:00000001:1.0:1786177134.797900:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306485504 00000100:00000001:1.0:1786177134.797903:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306481668 00000100:00000001:1.0:1786177134.797905:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306485504 00000100:00000001:1.0:1786177134.797908:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177134.797910:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.797913:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177134.797917:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177134/real 1786177134] req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177134.797927:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177134.797928:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177134.797931:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177134.797950:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177134.797953:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177134.797956:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177134.797960:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177134.797969:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.797977:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177134.797978:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.797980:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177134.797982:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177134.797986:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.797993:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177134.797997:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798005:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177134.798007:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.798009:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177134.798011:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.798013:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177134.798015:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177134.798018:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177134.798022:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177134.798025:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177134.798030:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177134.798039:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579648:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177134.798044:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177134.798048:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798055:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177134.798056:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177134.798059:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177134.798061:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177134.798063:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177134.798066:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579776:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177134.798069:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177134.798072:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177134.798073:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177134.798075:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177134.798079:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177134.798082:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177134.798086:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177134.798089:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177134.798092:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177134.798095:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235579776, portal 10 00000100:00000001:1.0:1786177134.798098:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177134.798099:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469623680 : -110371239927936 : ffff9b9e31735f80) 00000100:00000040:1.0:1786177134.798106:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798114:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177134.798117:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177134.798120:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177134.798122:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235579776, offset 0 00000400:00000200:1.0:1786177134.798128:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177134.798132:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177134.798136:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177134.798140:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177134.798143:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177134.798146:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177134.798149:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177134.798152:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426596:426596:256:4294967295] 192.168.202.132@tcp LPNI seq info [426558:426558:8:4294967295] 00000400:00000200:1.0:1786177134.798156:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177134.798163:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177134.798168:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177134.798171:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177134.798176:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177134.798179:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786177134.798184:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177134.798186:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1786177134.798191:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177134.798192:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177134.798196:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177134.798200:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177134.798203:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177134.798206:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177134.798208:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177134.798212:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798221:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177134.798224:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798231:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.798233:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177134.798236:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177134.798240:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177134.798243:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177134.798245:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306481668 00000100:00000001:1.0:1786177134.798248:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306485504 00000100:00000001:1.0:1786177134.798251:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306481668 00000100:00000001:1.0:1786177134.798254:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306485504 00000100:00000001:1.0:1786177134.798256:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177134.798258:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.798260:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177134.798265:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177134/real 1786177134] req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177134.798274:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177134.798276:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177134.798277:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177134.798280:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177134.798282:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177134.798284:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177134.798288:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177134.798297:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798305:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177134.798306:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.798309:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177134.798310:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177134.798314:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798321:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177134.798326:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798334:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177134.798336:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.798338:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177134.798339:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.798342:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177134.798344:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177134.798347:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177134.798351:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177134.798353:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177134.798378:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177134.798387:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579776:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177134.798392:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177134.798396:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798403:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177134.798405:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177134.798407:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177134.798409:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177134.798411:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177134.798414:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579904:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177134.798418:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177134.798420:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177134.798421:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177134.798424:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177134.798427:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177134.798429:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177134.798433:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177134.798436:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177134.798440:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177134.798444:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235579904, portal 4 00000100:00000001:1.0:1786177134.798446:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177134.798447:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469628160 : -110371239923456 : ffff9b9e31737100) 00000100:00000040:1.0:1786177134.798453:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798461:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177134.798464:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177134.798467:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177134.798469:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235579904, offset 0 00000400:00000200:1.0:1786177134.798474:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177134.798478:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177134.798482:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177134.798485:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177134.798488:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177134.798491:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177134.798493:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177134.798497:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426597:426597:256:4294967295] 192.168.202.132@tcp LPNI seq info [426559:426559:8:4294967295] 00000400:00000200:1.0:1786177134.798502:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177134.798510:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177134.798515:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177134.798519:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177134.798524:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177134.798527:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786177134.798531:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177134.798535:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1786177134.798539:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177134.798541:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177134.798545:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177134.798549:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177134.798552:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177134.798554:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177134.798557:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177134.798561:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798570:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177134.798573:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798586:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.798588:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177134.798592:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177134.798595:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177134.798598:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177134.798601:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306481669 00000100:00000001:1.0:1786177134.798603:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306485504 00000100:00000001:1.0:1786177134.798606:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306481669 00000100:00000001:1.0:1786177134.798608:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306485504 00000100:00000001:1.0:1786177134.798610:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177134.798613:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.798615:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177134.798620:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177134/real 1786177134] req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177134.798628:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177134.798630:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177134.798632:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177134.798634:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177134.798636:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177134.798639:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177134.798642:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177134.798652:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798659:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177134.798661:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.798663:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177134.798664:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177134.798668:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798676:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177134.798680:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798688:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177134.798691:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.798693:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177134.798694:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.798696:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177134.798698:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177134.798701:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177134.798705:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177134.798707:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177134.798712:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177134.798721:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235579904:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177134.798726:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177134.798730:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798737:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177134.798738:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177134.798741:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177134.798743:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177134.798745:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177134.798748:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580032:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177134.798752:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177134.798754:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177134.798756:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177134.798758:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177134.798762:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177134.798765:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177134.798768:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177134.798771:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177134.798775:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177134.798778:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235580032, portal 4 00000100:00000001:1.0:1786177134.798781:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177134.798782:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469618304 : -110371239933312 : ffff9b9e31734a80) 00000100:00000040:1.0:1786177134.798788:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798795:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177134.798798:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177134.798801:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177134.798803:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235580032, offset 0 00000400:00000200:1.0:1786177134.798808:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177134.798812:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177134.798816:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177134.798820:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177134.798823:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177134.798825:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177134.798828:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177134.798832:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426598:426598:256:4294967295] 192.168.202.132@tcp LPNI seq info [426560:426560:8:4294967295] 00000400:00000200:1.0:1786177134.798836:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177134.798843:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177134.798849:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177134.798852:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177134.798857:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177134.798860:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1786177134.798864:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177134.798867:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1786177134.798871:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177134.798874:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177134.798877:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177134.798880:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177134.798883:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177134.798885:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177134.798888:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177134.798892:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798901:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177134.798905:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177159 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.798913:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.798914:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177134.798918:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177134.798921:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177134.798925:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177134.798927:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306481671 00000100:00000001:1.0:1786177134.798930:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306485504 00000100:00000001:1.0:1786177134.798933:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306481671 00000100:00000001:1.0:1786177134.798952:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306485504 00000100:00000001:1.0:1786177134.798955:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177134.798957:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.798959:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177134.798964:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177134/real 1786177134] req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177134.798973:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177134.798975:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177134.798976:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177134.798979:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177134.798981:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177134.798984:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177134.798987:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177134.798996:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.799004:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177134.799005:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.799007:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177134.799009:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177134.799013:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.799021:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177134.799025:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.799033:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177134.799036:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.799038:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177134.799039:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.799041:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177134.799043:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177134.799046:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177134.799049:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177134.799052:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177134.799056:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177134.799064:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580032:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177134.799068:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.799071:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177134.799075:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736680 x1872930235579648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.799082:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177134.799084:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177134.799086:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177134.799089:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177134.799091:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177134.799092:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177134.799095:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177134.799097:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:1.0:1786177134.799100:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177134.799101:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177134.799103:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736680. 00000100:00000001:1.0:1786177134.799106:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177134.799108:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.799110:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177134.799114:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735f80 x1872930235579776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.799122:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177134.799123:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177134.799125:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177134.799127:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177134.799128:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177134.799130:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177134.799132:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177134.799134:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:1.0:1786177134.799137:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177134.799138:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177134.799140:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735f80. 00000100:00000001:1.0:1786177134.799142:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177134.799144:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.799146:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177134.799149:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737100 x1872930235579904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.799156:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177134.799157:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177134.799159:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177134.799162:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177134.799164:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177134.799165:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177134.799167:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177134.799168:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:1.0:1786177134.799171:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177134.799172:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177134.799173:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737100. 00000100:00000001:1.0:1786177134.799176:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177134.799177:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.799179:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177134.799183:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734a80 x1872930235580032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177159 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177134.799191:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177134.799192:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177134.799194:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177134.799196:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177134.799198:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177134.799199:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177134.799202:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177134.799203:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:1.0:1786177134.799205:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177134.799207:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177134.799208:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734a80. 00000100:00000001:1.0:1786177134.799211:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177134.799212:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.799214:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177134.799218:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177134.799219:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177134.799222:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177134.799223:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177135.308486:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177135.308493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177135.308495:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177135.308500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177135.308504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177135.308507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177135.308509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177135.308511:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177135.308514:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177135.692437:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177135.692443:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177135.692443:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177135.692451:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177135.692451:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177135.692455:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177135.692457:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177135.692458:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177135.692461:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177135.692464:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177135.692471:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177135.692475:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1786177135.692479:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000100:00000001:1.0:1786177135.692479:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1786177135.692481:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786177135.692484:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177135.692485:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177135.692490:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177136.332461:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177136.332467:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177136.332469:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177136.332475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177136.332478:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177136.332482:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177136.332484:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177136.332487:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177136.332489:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177136.716469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177136.716472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177136.716476:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177136.716476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177136.716478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177136.716482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177136.716482:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177136.716483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177136.716486:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177136.716487:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177136.716488:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177136.716489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177136.716492:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177136.716495:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177136.716497:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177136.716548:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177136.716553:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177136.716557:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177137.356542:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177137.356549:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177137.356551:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177137.356556:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177137.356559:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177137.356562:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177137.356564:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177137.356566:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177137.356568:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177137.740420:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177137.740427:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177137.740428:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177137.740430:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177137.740434:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177137.740434:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177137.740439:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177137.740441:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177137.740443:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177137.740444:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177137.740448:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177137.740450:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177137.740451:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177137.740452:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177137.740455:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177137.740478:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177137.740480:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177137.740482:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177138.380470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177138.380479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177138.380482:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177138.380488:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177138.380494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177138.380504:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177138.380506:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177138.380508:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177138.380510:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177138.700459:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306485504, at 4306485568 00000100:00000040:1.0:1786177138.700459:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306485504, at 4306485568 00000100:00000010:0.0:1786177138.700475:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c625a0. 00000100:00000010:1.0:1786177138.700476:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c626c0. 00000100:00000001:1.0:1786177138.764443:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177138.764447:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177138.764450:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177138.764451:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177138.764452:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177138.764456:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177138.764457:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177138.764457:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177138.764460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177138.764460:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177138.764462:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177138.764464:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177138.764466:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177138.764469:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177138.764472:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177138.764508:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177138.764512:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177138.764514:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177139.404451:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177139.404459:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177139.404460:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177139.404465:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177139.404468:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177139.404471:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177139.404473:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177139.404476:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177139.404479:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177139.788444:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177139.788450:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177139.788454:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177139.788457:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177139.788459:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177139.788462:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.788463:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177139.788465:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177139.788468:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.788468:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177139.788472:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.788472:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177139.788477:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177139.788478:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177139.788481:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177139.788483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.788483:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177139.788487:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.788490:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177139.788492:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.788494:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177139.788531:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177139.788535:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177139.788537:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177139.916465:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177139.916473:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177139.916476:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177139.916479:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177139.916483:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177139.916489:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11814 00000400:00000200:2.0:1786177139.916496:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177139.916501:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177139.916504:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177139.916507:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916509:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177139.916511:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916513:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177139.916516:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177139.916517:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177139.916520:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177139.916522:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177139.916524:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177139.916525:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177139.916528:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916530:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177139.916531:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916532:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177139.916534:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177139.916537:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916538:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177139.916540:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177139.916543:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177139.916545:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.916546:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177139.916548:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177139.916557:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734e00. 00000020:00000040:2.0:1786177139.916560:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177139.916565:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177139.916566:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.916570:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177139.916573:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177139.916577:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177139.916586:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:2.0:1786177139.916593:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177139.916603:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177139.916614:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177139.916619:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177139.916624:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001f62212] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177139.916657:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177139.916660:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177139.916664:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177139.916665:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177139.916667:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177139.916670:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177139.916673:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11814 00000400:00000200:2.0:1786177139.916678:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177139.916681:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177139.916684:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177139.916687:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916690:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177139.916692:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916694:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177139.916696:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177139.916697:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177139.916699:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177139.916701:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177139.916702:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177139.916704:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177139.916706:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177139.916708:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177139.916708:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177139.916712:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916714:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177139.916715:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1786177139.916716:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177139.916720:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177139.916721:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1786177139.916722:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177139.916725:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177139.916730:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1786177139.916733:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177139.916733:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.916737:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177139.916741:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177139.916746:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.916750:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.916754:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000010:2.0:1786177139.916756:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735180. 00000100:00000001:1.0:1786177139.916759:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177139.916760:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177139.916765:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177139.916767:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.916769:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177139.916770:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177139.916771:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177139.916774:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 02000000:00000001:1.0:1786177139.916777:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1786177139.916777:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177139.916782:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177139.916786:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177139.916795:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.916795:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177139.916800:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177139.916805:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235580160:192.168.202.132@tcp:250:kworker.0 00000100:00000040:2.0:1786177139.916810:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ba0f078] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.916813:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177139.916818:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177139.916822:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.916825:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177139.916827:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1786177139.916831:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:1.0:1786177139.916835:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261aec00. 00000100:00080000:2.0:1786177139.916837:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:1.0:1786177139.916840:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.916841:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177139.916845:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177139.916849:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177139.916850:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:2.0:1786177139.916854:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11814 00000400:00000010:1.0:1786177139.916857:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000200:2.0:1786177139.916863:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177139.916867:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000400:00000200:2.0:1786177139.916872:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177139.916875:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000200:1.0:1786177139.916877:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235580160, portal 25 00000100:00000040:2.0:1786177139.916880:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177139.916882:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177139.916898:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177139.916902:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916904:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177139.916906:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177139.916908:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177139.916910:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177139.916912:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177139.916915:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177139.916917:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177139.916920:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916922:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177139.916924:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916926:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177139.916929:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177139.916937:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177139.916940:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177139.916941:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177139.916945:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177139.916947:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.916949:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177139.916951:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177139.916953:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.916955:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177139.916960:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000020:00000040:2.0:1786177139.916962:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177139.916965:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177139.916966:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.916968:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177139.916969:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177139.916971:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177139.916975:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab400. 00000100:00000001:2.0:1786177139.916979:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177139.916985:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177139.916993:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177139.916995:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177139.917000:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177139.917008:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177139.917010:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177139.917012:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177139.917013:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177139.917015:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177139.917017:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177139.917019:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11814 00000100:00000001:1.0:1786177139.917022:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469619200 : -110371239932416 : ffff9b9e31734e00) 00000400:00000200:2.0:1786177139.917023:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177139.917031:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177139.917035:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177139.917036:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177139.917043:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.917050:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177139.917056:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.917063:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177139.917069:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177139.917070:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:2.0:1786177139.917072:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177139.917077:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1786177139.917078:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000020:00000040:2.0:1786177139.917081:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000400:00000010:1.0:1786177139.917084:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000020:00000001:2.0:1786177139.917085:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177139.917089:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000200:1.0:1786177139.917091:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235580160, offset 0 00000100:00000040:2.0:1786177139.917094:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.917097:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177139.917099:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.917100:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177139.917105:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000020:00000040:2.0:1786177139.917105:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177139.917110:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177139.917113:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177139.917115:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1786177139.917118:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177139.917119:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177139.917123:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.917125:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177139.917127:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000200:1.0:1786177139.917129:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000008:00000001:2.0:1786177139.917131:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.917135:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177139.917138:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:2.0:1786177139.917142:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000400:00000200:1.0:1786177139.917145:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000040:2.0:1786177139.917147:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000400:00000200:1.0:1786177139.917152:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1786177139.917154:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1786177139.917158:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 02000000:00000001:2.0:1786177139.917158:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.917164:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1786177139.917167:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426599:426599:256:4294967295] 192.168.202.132@tcp LPNI seq info [426561:426561:8:4294967295] 02000000:00000001:2.0:1786177139.917168:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177139.917172:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177139.917177:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:2.0:1786177139.917187:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000400:00000200:1.0:1786177139.917189:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177139.917194:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177139.917200:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00100000:2.0:1786177139.917205:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1786177139.917209:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000100:00000001:2.0:1786177139.917220:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.0:1786177139.917222:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177139.917224:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177139.917228:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000040:2.0:1786177139.917234:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1786177139.917241:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177139.917249:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1786177139.917253:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1786177139.917259:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00080000:2.0:1786177139.917260:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1786177139.917265:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000100:00080000:2.0:1786177139.917265:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177139.917268:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177139.917271:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177139.917273:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177139.917274:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1786177139.917277:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11814 00000400:00000200:2.0:1786177139.917284:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177139.917287:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177139.917291:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177139.917294:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177139.917297:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000100:00000040:2.0:1786177139.917298:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177139.917303:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:2.0:1786177139.917303:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177139.917310:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1786177139.917314:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.917317:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000200:1.0:1786177139.917319:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177139.917320:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177139.917322:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177139.917325:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177139.917328:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177139.917332:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177139.917336:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177139.917343:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177139.917344:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1786177139.917345:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177139.917349:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177139.917352:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:1.0:1786177139.917354:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177139.917354:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177139.917385:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177139.917387:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177139.917388:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177139.917390:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177139.917391:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.917394:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177139.917398:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177139.917403:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.917408:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177139.917413:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000020:00000040:2.0:1786177139.917418:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177139.917423:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177139.917425:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.917426:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.917427:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177139.917430:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177139.917432:0:2392:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1786177139.917433:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177139.917438:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:1.0:1786177139.917442:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177139.917443:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177139.917454:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177139.917455:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177139.917461:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306490625 00000100:00000001:2.0:1786177139.917464:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177139.917467:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177139.917470:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c626c0. 00000100:00000040:2.0:1786177139.917470:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.917479:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177139.917487:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177139.917491:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11824) 00000100:00000001:1.0:1786177139.917492:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306490625 00000100:00000010:1.0:1786177139.917498:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62540. 00000100:00000001:1.0:1786177139.917502:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177139.917508:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177139.917512:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.917515:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177139.917521:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177139/real 1786177139] req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177139.917544:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177139.917546:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177139.917548:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177139.917551:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177139.917553:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177139.917556:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177139.917560:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177139.917569:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.917577:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177139.917578:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.917580:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177139.917582:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177139.917587:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.917594:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177139.917598:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.917605:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177139.917608:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.917610:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177139.917611:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.917614:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177139.917616:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177139.917620:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177139.917624:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177139.917627:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177139.917631:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177139.917641:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235580160:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177139.917646:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.917648:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177139.917652:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734e00 x1872930235580160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.917658:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177139.917661:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177139.917664:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177139.917667:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177139.917669:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177139.917671:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177139.917673:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177139.917675:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:1.0:1786177139.917680:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177139.917682:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177139.917683:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734e00. 00000100:00000001:1.0:1786177139.917687:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177139.917688:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.917690:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.917697:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177139.917700:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.917702:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177139.917703:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177139.917705:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177139.917709:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.917716:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177139.917717:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177139.917719:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177139.917721:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177139.917724:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177139.917727:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580288:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177139.917730:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177139.917732:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177139.917734:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177139.917735:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177139.917738:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177139.917740:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177139.917743:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177139.917747:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177139.917750:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177139.917753:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235580288, portal 10 00000100:00000001:1.0:1786177139.917755:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177139.917757:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620096 : -110371239931520 : ffff9b9e31735180) 00000100:00000040:1.0:1786177139.917763:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.917770:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177139.917773:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177139.917775:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177139.917778:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235580288, offset 0 00000400:00000200:1.0:1786177139.917783:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177139.917787:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177139.917791:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177139.917794:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177139.917797:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177139.917799:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177139.917802:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177139.917805:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426600:426600:256:4294967295] 192.168.202.132@tcp LPNI seq info [426562:426562:8:4294967295] 00000400:00000200:1.0:1786177139.917810:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177139.917817:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177139.917822:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177139.917826:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177139.917830:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177139.917833:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177139.917837:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177139.917840:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177139.917845:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177139.917847:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177139.917850:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177139.917853:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177139.917856:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177139.917859:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177139.917861:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177139.917865:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.917874:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177139.917877:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.917884:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.917904:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177139.917909:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177139.917913:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177139.917917:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177139.917919:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306486788 00000100:00000001:1.0:1786177139.917921:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306490625 00000100:00000001:1.0:1786177139.917924:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306486788 00000100:00000001:1.0:1786177139.917926:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306490625 00000100:00000001:1.0:1786177139.917928:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177139.917930:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.917933:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177139.917937:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177139/real 1786177139] req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177139.917945:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177139.917947:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177139.917949:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177139.917951:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177139.917953:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177139.917956:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177139.917959:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177139.917968:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.917975:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177139.917976:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.917978:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177139.917980:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177139.917984:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.917991:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177139.917995:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918003:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177139.918006:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918007:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177139.918009:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918011:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177139.918013:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177139.918016:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177139.918020:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177139.918022:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177139.918027:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177139.918036:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580288:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177139.918040:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177139.918044:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918051:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177139.918053:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177139.918055:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177139.918057:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177139.918059:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177139.918063:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580416:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177139.918067:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177139.918069:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177139.918070:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177139.918072:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177139.918076:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177139.918078:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177139.918082:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177139.918085:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177139.918088:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177139.918091:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235580416, portal 10 00000100:00000001:1.0:1786177139.918093:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177139.918094:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469618304 : -110371239933312 : ffff9b9e31734a80) 00000100:00000040:1.0:1786177139.918101:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918108:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177139.918110:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177139.918113:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177139.918115:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235580416, offset 0 00000400:00000200:1.0:1786177139.918120:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177139.918124:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177139.918128:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177139.918131:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177139.918134:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177139.918137:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177139.918139:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177139.918142:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426601:426601:256:4294967295] 192.168.202.132@tcp LPNI seq info [426563:426563:8:4294967295] 00000400:00000200:1.0:1786177139.918147:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177139.918153:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177139.918158:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177139.918162:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177139.918165:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177139.918168:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177139.918172:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177139.918175:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177139.918179:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177139.918181:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177139.918184:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177139.918188:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177139.918192:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177139.918194:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177139.918197:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177139.918201:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918209:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177139.918212:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918219:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.918220:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177139.918224:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177139.918227:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177139.918230:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177139.918232:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306486788 00000100:00000001:1.0:1786177139.918235:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306490625 00000100:00000001:1.0:1786177139.918237:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306486788 00000100:00000001:1.0:1786177139.918240:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306490625 00000100:00000001:1.0:1786177139.918242:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177139.918243:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.918245:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177139.918249:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177139/real 1786177139] req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177139.918257:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177139.918259:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177139.918261:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177139.918264:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177139.918265:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177139.918267:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177139.918271:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177139.918279:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918286:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177139.918288:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918289:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177139.918291:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177139.918294:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918301:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177139.918306:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918312:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177139.918315:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918316:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177139.918318:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918320:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177139.918322:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177139.918324:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177139.918327:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177139.918330:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177139.918334:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177139.918342:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580416:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177139.918347:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177139.918350:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918378:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177139.918379:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177139.918382:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177139.918384:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177139.918386:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177139.918389:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580544:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177139.918393:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177139.918395:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177139.918396:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177139.918398:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177139.918402:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177139.918404:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177139.918407:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177139.918410:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177139.918414:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177139.918417:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235580544, portal 4 00000100:00000001:1.0:1786177139.918419:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177139.918420:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469628160 : -110371239923456 : ffff9b9e31737100) 00000100:00000040:1.0:1786177139.918426:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918432:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177139.918435:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177139.918437:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177139.918439:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235580544, offset 0 00000400:00000200:1.0:1786177139.918444:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177139.918448:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177139.918452:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177139.918455:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177139.918458:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177139.918461:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177139.918463:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177139.918467:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426602:426602:256:4294967295] 192.168.202.132@tcp LPNI seq info [426564:426564:8:4294967295] 00000400:00000200:1.0:1786177139.918471:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177139.918478:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177139.918483:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177139.918487:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177139.918491:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177139.918494:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177139.918498:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177139.918502:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177139.918506:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177139.918508:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177139.918511:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177139.918515:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177139.918518:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177139.918520:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177139.918522:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177139.918526:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918534:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177139.918538:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918545:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.918547:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177139.918550:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177139.918553:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177139.918556:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177139.918558:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306486789 00000100:00000001:1.0:1786177139.918561:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306490625 00000100:00000001:1.0:1786177139.918564:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306486789 00000100:00000001:1.0:1786177139.918566:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306490625 00000100:00000001:1.0:1786177139.918568:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177139.918570:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.918572:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177139.918576:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177139/real 1786177139] req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177139.918585:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177139.918586:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177139.918588:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177139.918591:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177139.918592:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177139.918594:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177139.918597:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177139.918607:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918613:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177139.918615:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918617:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177139.918618:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177139.918622:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918629:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177139.918633:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918640:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177139.918643:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918644:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177139.918646:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918648:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177139.918649:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177139.918652:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177139.918656:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177139.918658:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177139.918662:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177139.918670:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580544:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177139.918674:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177139.918677:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918683:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177139.918685:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177139.918687:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177139.918688:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177139.918690:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177139.918693:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580672:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177139.918696:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177139.918698:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177139.918699:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177139.918701:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177139.918704:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177139.918706:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177139.918709:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177139.918711:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177139.918714:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177139.918717:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235580672, portal 4 00000100:00000001:1.0:1786177139.918718:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177139.918720:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469623680 : -110371239927936 : ffff9b9e31735f80) 00000100:00000040:1.0:1786177139.918725:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918731:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177139.918734:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177139.918736:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177139.918738:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235580672, offset 0 00000400:00000200:1.0:1786177139.918744:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177139.918748:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177139.918752:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177139.918754:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177139.918757:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177139.918759:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177139.918762:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177139.918765:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426603:426603:256:4294967295] 192.168.202.132@tcp LPNI seq info [426565:426565:8:4294967295] 00000400:00000200:1.0:1786177139.918769:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177139.918775:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177139.918779:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177139.918783:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177139.918786:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177139.918789:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177139.918794:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177139.918797:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177139.918801:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177139.918803:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177139.918806:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177139.918809:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177139.918812:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177139.918814:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177139.918816:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177139.918820:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918828:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177139.918831:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177164 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918838:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.918840:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177139.918843:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177139.918846:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177139.918849:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177139.918852:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306486791 00000100:00000001:1.0:1786177139.918855:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306490625 00000100:00000001:1.0:1786177139.918857:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306486791 00000100:00000001:1.0:1786177139.918859:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306490625 00000100:00000001:1.0:1786177139.918861:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177139.918863:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.918865:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177139.918869:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177139/real 1786177139] req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177139.918877:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177139.918878:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177139.918880:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177139.918882:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177139.918907:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177139.918911:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177139.918920:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177139.918929:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918936:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177139.918938:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918940:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177139.918941:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177139.918945:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918952:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177139.918956:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.918963:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177139.918966:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918968:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177139.918969:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918971:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177139.918973:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177139.918975:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177139.918978:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177139.918980:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177139.918984:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177139.918992:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580672:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177139.918996:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.918998:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177139.919001:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735180 x1872930235580288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.919008:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177139.919010:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177139.919011:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:1.0:1786177139.919014:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177139.919016:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177139.919017:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177139.919019:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177139.919021:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:1.0:1786177139.919024:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177139.919025:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177139.919027:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735180. 00000100:00000001:1.0:1786177139.919029:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177139.919031:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.919032:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177139.919036:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734a80 x1872930235580416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.919049:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177139.919051:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177139.919052:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:1.0:1786177139.919055:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177139.919056:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177139.919057:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177139.919059:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177139.919061:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:1.0:1786177139.919063:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177139.919064:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177139.919066:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734a80. 00000100:00000001:1.0:1786177139.919068:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177139.919069:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.919071:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177139.919074:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737100 x1872930235580544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.919081:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177139.919082:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177139.919084:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af800. 02000000:00000001:1.0:1786177139.919086:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177139.919088:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177139.919089:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177139.919091:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177139.919093:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:1.0:1786177139.919095:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177139.919096:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177139.919097:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737100. 00000100:00000001:1.0:1786177139.919100:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177139.919101:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.919103:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177139.919106:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735f80 x1872930235580672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177164 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177139.919113:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177139.919115:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177139.919116:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177139.919119:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177139.919120:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177139.919121:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177139.919123:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177139.919125:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:1.0:1786177139.919127:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177139.919128:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177139.919129:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735f80. 00000100:00000001:1.0:1786177139.919132:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177139.919133:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.919136:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177139.919139:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177139.919141:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177139.919143:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177139.919145:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177140.428457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177140.428464:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177140.428466:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177140.428470:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177140.428474:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177140.428477:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177140.428479:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177140.428482:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177140.428484:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177140.812451:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177140.812452:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177140.812457:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177140.812459:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177140.812463:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177140.812464:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177140.812467:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177140.812467:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177140.812473:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177140.812474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177140.812477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177140.812478:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177140.812481:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177140.812483:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177140.812485:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177140.812522:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177140.812526:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177140.812529:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177141.452468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177141.452477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177141.452479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177141.452485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177141.452489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177141.452493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177141.452496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177141.452498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177141.452501:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177141.836445:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177141.836455:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177141.836457:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177141.836464:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177141.836465:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177141.836469:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177141.836471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177141.836474:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177141.836477:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177141.836479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177141.836480:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177141.836483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177141.836485:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177141.836492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177141.836495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177141.836578:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177141.836582:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177141.836585:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177142.476509:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177142.476518:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177142.476520:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177142.476526:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177142.476530:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177142.476534:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177142.476536:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177142.476539:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177142.476541:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177142.860505:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177142.860509:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177142.860516:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177142.860517:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177142.860520:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177142.860522:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1786177142.860528:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177142.860529:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177142.860536:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177142.860537:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177142.860541:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177142.860542:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177142.860548:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177142.860548:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177142.860555:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177142.860555:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177142.860563:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177142.860564:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177142.860568:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177142.860573:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177142.860574:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177142.860576:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177142.860577:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177142.860579:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177142.860580:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177142.860582:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177142.860583:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177142.860658:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177142.860662:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177142.860665:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177143.500444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177143.500452:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177143.500454:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177143.500460:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177143.500464:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177143.500467:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177143.500471:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177143.500475:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177143.500478:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177143.820474:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306490625, at 4306490688 00000100:00000040:1.0:1786177143.820474:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306490625, at 4306490688 00000100:00000010:1.0:1786177143.820489:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62540. 00000100:00000010:0.0:1786177143.820495:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c626c0. 00000100:00000001:1.0:1786177143.884475:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177143.884480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177143.884483:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177143.884484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177143.884487:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177143.884490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177143.884492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177143.884494:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177143.884495:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177143.884498:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177143.884500:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177143.884505:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177143.884507:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177143.884510:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177143.884513:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177143.884557:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177143.884560:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177143.884563:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177144.524440:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177144.524448:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177144.524451:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177144.524457:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177144.524462:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177144.524465:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177144.524467:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177144.524469:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177144.524472:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177144.908439:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177144.908446:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177144.908447:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177144.908452:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177144.908455:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177144.908456:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177144.908459:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177144.908460:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177144.908463:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177144.908466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177144.908467:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177144.908468:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177144.908471:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177144.908473:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177144.908475:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177144.908525:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177144.908529:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177144.908533:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177145.036480:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177145.036489:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177145.036492:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177145.036494:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177145.036498:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177145.036504:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11819 00000400:00000200:1.0:1786177145.036511:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177145.036516:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177145.036518:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177145.036521:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036524:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177145.036526:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036528:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177145.036531:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177145.036532:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177145.036534:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177145.036537:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177145.036539:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177145.036541:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177145.036544:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036545:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177145.036546:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036548:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177145.036550:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177145.036552:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036554:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177145.036556:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177145.036558:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177145.036560:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.036562:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177145.036563:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177145.036571:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8e00. 00000020:00000040:1.0:1786177145.036574:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177145.036577:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177145.036579:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.036581:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177145.036583:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177145.036587:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177145.036594:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad400. 00000100:00000001:1.0:1786177145.036599:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177145.036605:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177145.036613:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177145.036617:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177145.036620:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ba93daf1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177145.036640:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177145.036643:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177145.036645:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177145.036647:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177145.036648:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177145.036651:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177145.036652:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11819 00000400:00000200:1.0:1786177145.036655:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177145.036657:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177145.036659:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177145.036662:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036664:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177145.036666:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036668:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177145.036670:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177145.036671:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177145.036672:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177145.036674:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177145.036676:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177145.036676:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1786177145.036679:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177145.036682:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036683:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177145.036684:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036686:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177145.036687:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:1.0:1786177145.036689:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036690:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177145.036693:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177145.036694:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1786177145.036698:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177145.036698:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1786177145.036702:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.036705:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:2.0:1786177145.036707:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1786177145.036708:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177145.036714:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.036718:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177145.036724:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8700. 00000100:00000001:2.0:1786177145.036725:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1786177145.036729:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000100:00000001:2.0:1786177145.036730:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177145.036735:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177145.036736:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177145.036737:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.036740:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177145.036740:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177145.036743:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177145.036745:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.036747:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177145.036750:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261aec00. 00000100:00000001:1.0:1786177145.036755:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177145.036755:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235580800:192.168.202.132@tcp:250:kworker.0 00000100:00100000:1.0:1786177145.036765:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177145.036765:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177145.036770:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1786177145.036774:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1786177145.036775:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.036778:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.036782:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1786177145.036783:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007a756761] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1786177145.036789:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:1.0:1786177145.036791:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177145.036794:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177145.036799:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:2.0:1786177145.036799:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.036804:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177145.036805:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177145.036807:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177145.036809:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177145.036813:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11819 00000400:00000010:2.0:1786177145.036815:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000200:1.0:1786177145.036819:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177145.036822:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000010:2.0:1786177145.036823:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00080000:1.0:1786177145.036825:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177145.036829:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036831:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1786177145.036832:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235580800, portal 25 00000100:00000040:1.0:1786177145.036833:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177145.036851:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177145.036853:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487985664 : -110371221565952 : ffff9b9e328b8e00) 00000100:00000001:1.0:1786177145.036859:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:2.0:1786177145.036860:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177145.036864:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177145.036867:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177145.036871:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177145.036875:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000100:00000001:2.0:1786177145.036876:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1786177145.036881:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177145.036882:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177145.036885:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000020:00000001:1.0:1786177145.036888:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000200:2.0:1786177145.036891:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235580800, offset 0 00000100:00000040:1.0:1786177145.036896:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177145.036902:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036903:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177145.036908:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.036912:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177145.036913:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:1.0:1786177145.036919:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:1.0:1786177145.036923:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:2.0:1786177145.036926:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:1.0:1786177145.036930:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177145.036933:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1786177145.036935:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1786177145.036936:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177145.036940:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.036943:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:2.0:1786177145.036943:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000008:00100000:1.0:1786177145.036947:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177145.036952:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177145.036952:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1786177145.036958:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177145.036961:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000010:1.0:1786177145.036962:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9c00. 00000020:00000040:1.0:1786177145.036966:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000200:2.0:1786177145.036968:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426604:426604:256:4294967295] 192.168.202.132@tcp LPNI seq info [426566:426566:8:4294967295] 02000000:00000001:1.0:1786177145.036969:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177145.036972:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.036975:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177145.036978:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1786177145.036978:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177145.036983:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177145.036987:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af000. 00000100:00000001:1.0:1786177145.036992:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177145.036993:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00100000:1.0:1786177145.037001:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.0:1786177145.037003:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177145.037010:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177145.037012:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177145.037019:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3ea07ed] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:2.0:1786177145.037020:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00000001:1.0:1786177145.037029:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177145.037032:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:1.0:1786177145.037034:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1786177145.037038:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786177145.037045:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:1.0:1786177145.037047:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:2.1:1786177145.037049:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:1.0:1786177145.037050:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177145.037055:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1786177145.037059:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177145.037061:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000800:00000010:2.0:1786177145.037062:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000100:00080000:1.0:1786177145.037066:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11819 00000400:00000200:2.0:1786177145.037070:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177145.037071:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177145.037077:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:2.0:1786177145.037078:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00080000:1.0:1786177145.037081:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177145.037084:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000100:00000040:1.0:1786177145.037086:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:2.0:1786177145.037088:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:1.0:1786177145.037089:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177145.037093:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1786177145.037094:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.037098:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177145.037102:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000200:2.0:1786177145.037102:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177145.037103:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177145.037107:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177145.037111:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177145.037117:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:2.0:1786177145.037117:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177145.037123:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177145.037124:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:1.0:1786177145.037135:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177145.037135:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.037138:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177145.037142:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177145.037147:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177145.037150:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177145.037151:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306495744 00000100:00000040:1.0:1786177145.037157:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000010:2.0:1786177145.037158:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c4e0. 00000100:00000001:1.0:1786177145.037163:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177145.037166:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:2.0:1786177145.037166:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1786177145.037172:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177145.037175:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.037175:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306495744 02000000:00000001:1.0:1786177145.037181:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:2.0:1786177145.037182:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cae0. 02000000:00000001:1.0:1786177145.037184:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177145.037186:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1786177145.037190:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.037191:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177145.037193:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177145.037197:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1786177145.037197:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.037201:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000008:00000001:1.0:1786177145.037202:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.037204:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177145.037207:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177145/real 1786177145] req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1786177145.037210:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328ba680. 00000020:00000040:1.0:1786177145.037215:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000400:00000001:2.0:1786177145.037220:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177145.037222:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1786177145.037226:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:2.0:1786177145.037226:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177145.037229:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 02000000:00000001:1.0:1786177145.037230:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177145.037231:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177145.037236:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1786177145.037237:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:2.0:1786177145.037240:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177145.037242:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177145.037246:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177145.037251:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177145.037254:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad800. 00000100:00000001:1.0:1786177145.037262:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.037264:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177145.037267:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.037269:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:1.0:1786177145.037273:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177145.037278:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177145.037281:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177145.037284:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177145.037289:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177145.037290:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000497a117c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.037313:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177145.037315:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177145.037318:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1786177145.037320:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177145.037322:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177145.037325:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177145.037328:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177145.037334:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177145.037335:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786177145.037339:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1786177145.037341:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11819 00000100:00000001:2.0:1786177145.037342:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177145.037346:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1786177145.037350:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177145.037350:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177145.037355:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177145.037379:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177145.037384:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177145.037387:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:1.0:1786177145.037390:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177145.037393:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177145.037397:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.037399:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177145.037400:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.037403:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:2.0:1786177145.037404:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177145.037405:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177145.037408:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177145.037411:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177145.037414:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:1.0:1786177145.037418:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:1.0:1786177145.037423:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:1.0:1786177145.037428:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.037431:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1786177145.037432:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235580800:192.168.202.132@tcp:250:kworker.0 00000100:00000040:1.0:1786177145.037434:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177145.037437:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177145.037442:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:1.0:1786177145.037446:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177145.037446:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177145.037452:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.037453:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1786177145.037454:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177145.037458:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177145.037462:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177145.037462:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8e00 x1872930235580800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177145.037463:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177145.037466:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177145.037471:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.037475:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177145.037480:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb100. 00000100:00000001:2.0:1786177145.037483:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177145.037486:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177145.037489:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa400. 00000020:00000040:1.0:1786177145.037492:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177145.037498:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177145.037499:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.037500:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177145.037501:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177145.037504:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.037505:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000010:1.0:1786177145.037508:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af800. 00000020:00000001:2.0:1786177145.037509:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:1.0:1786177145.037514:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177145.037514:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177145.037520:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00100000:1.0:1786177145.037521:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1786177145.037524:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad400. 00000100:00000001:1.0:1786177145.037531:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1786177145.037532:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1786177145.037533:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.037537:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:1.0:1786177145.037538:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f57195c8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1786177145.037542:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8e00. 00000100:00000001:1.0:1786177145.037550:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.037550:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177145.037554:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177145.037557:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11829) 00000100:00000001:2.0:1786177145.037559:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.037569:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177145.037572:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.037576:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177145.037578:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177145.037580:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177145.037586:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.037594:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177145.037596:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.037598:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177145.037600:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177145.037603:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177145.037606:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580928:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177145.037609:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177145.037612:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177145.037613:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.037616:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177145.037619:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa400. 02000000:00000001:2.0:1786177145.037621:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177145.037626:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177145.037629:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177145.037632:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177145.037636:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235580928, portal 10 00000100:00000001:2.0:1786177145.037639:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177145.037641:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487983872 : -110371221567744 : ffff9b9e328b8700) 00000100:00000040:2.0:1786177145.037648:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.037655:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177145.037658:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177145.037661:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177145.037665:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235580928, offset 0 00000400:00000200:2.0:1786177145.037670:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177145.037676:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177145.037681:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177145.037686:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177145.037689:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177145.037692:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177145.037694:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177145.037698:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426605:426605:256:4294967295] 192.168.202.132@tcp LPNI seq info [426567:426567:8:4294967295] 00000400:00000200:2.0:1786177145.037702:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177145.037712:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177145.037718:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177145.037722:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177145.037726:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177145.037729:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786177145.037734:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177145.037737:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1786177145.037741:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177145.037743:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177145.037747:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177145.037750:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177145.037754:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177145.037756:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177145.037758:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177145.037762:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.037777:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177145.037780:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.037787:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.037789:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177145.037792:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177145.037796:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177145.037799:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177145.037802:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306491908 00000100:00000001:2.0:1786177145.037804:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306495744 00000100:00000001:2.0:1786177145.037807:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306491908 00000100:00000001:2.0:1786177145.037809:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306495744 00000100:00000001:2.0:1786177145.037811:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177145.037814:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.037816:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177145.037820:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177145/real 1786177145] req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177145.037828:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177145.037829:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177145.037831:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177145.037834:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177145.037849:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177145.037852:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177145.037856:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177145.037865:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.037871:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177145.037873:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.037875:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177145.037877:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177145.037881:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.037888:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177145.037891:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.037899:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177145.037902:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.037903:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177145.037905:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.037907:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177145.037909:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177145.037912:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177145.037915:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177145.037918:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177145.037922:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177145.037931:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235580928:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177145.037936:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177145.037939:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.037946:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177145.037948:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.037950:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177145.037952:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177145.037954:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177145.037958:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581056:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177145.037962:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177145.037964:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177145.037966:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.037968:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177145.037972:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177145.037974:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177145.037978:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177145.037981:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177145.037984:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177145.037987:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235581056, portal 10 00000100:00000001:2.0:1786177145.037989:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177145.037991:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487989248 : -110371221562368 : ffff9b9e328b9c00) 00000100:00000040:2.0:1786177145.037997:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038004:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177145.038007:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177145.038009:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177145.038012:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235581056, offset 0 00000400:00000200:2.0:1786177145.038017:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177145.038021:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177145.038025:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177145.038029:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177145.038031:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177145.038034:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177145.038036:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177145.038040:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426606:426606:256:4294967295] 192.168.202.132@tcp LPNI seq info [426568:426568:8:4294967295] 00000400:00000200:2.0:1786177145.038044:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177145.038051:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177145.038056:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177145.038059:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177145.038064:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177145.038066:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786177145.038070:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177145.038073:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1786177145.038077:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177145.038079:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177145.038082:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177145.038086:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177145.038089:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177145.038091:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177145.038094:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177145.038098:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038106:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177145.038109:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038116:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.038118:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177145.038121:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177145.038124:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177145.038128:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177145.038130:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306491908 00000100:00000001:2.0:1786177145.038133:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306495744 00000100:00000001:2.0:1786177145.038135:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306491908 00000100:00000001:2.0:1786177145.038137:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306495744 00000100:00000001:2.0:1786177145.038139:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177145.038141:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.038143:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177145.038147:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177145/real 1786177145] req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177145.038156:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177145.038158:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177145.038160:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177145.038162:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177145.038164:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177145.038166:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177145.038170:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177145.038179:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038185:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177145.038186:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038188:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177145.038189:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177145.038193:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038200:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177145.038203:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038210:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177145.038213:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038215:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177145.038216:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038218:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177145.038220:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177145.038222:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177145.038225:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177145.038228:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177145.038232:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177145.038246:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581056:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177145.038250:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177145.038253:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038260:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177145.038262:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.038264:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177145.038266:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177145.038268:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177145.038271:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581184:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177145.038274:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177145.038276:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177145.038277:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.038279:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177145.038283:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786177145.038285:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177145.038289:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177145.038291:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177145.038294:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177145.038297:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235581184, portal 4 00000100:00000001:2.0:1786177145.038299:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177145.038301:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487991936 : -110371221559680 : ffff9b9e328ba680) 00000100:00000040:2.0:1786177145.038307:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038313:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177145.038315:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177145.038318:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177145.038320:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235581184, offset 0 00000400:00000200:2.0:1786177145.038324:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177145.038328:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177145.038332:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177145.038335:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177145.038337:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177145.038340:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177145.038343:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177145.038346:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426607:426607:256:4294967295] 192.168.202.132@tcp LPNI seq info [426569:426569:8:4294967295] 00000400:00000200:2.0:1786177145.038350:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177145.038376:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177145.038383:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177145.038387:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177145.038390:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177145.038393:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786177145.038397:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177145.038400:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1786177145.038405:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177145.038407:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177145.038410:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177145.038414:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177145.038416:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177145.038418:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177145.038421:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177145.038425:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038433:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177145.038436:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038443:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.038445:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177145.038448:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177145.038451:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177145.038454:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177145.038457:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306491909 00000100:00000001:2.0:1786177145.038459:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306495744 00000100:00000001:2.0:1786177145.038461:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306491909 00000100:00000001:2.0:1786177145.038463:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306495744 00000100:00000001:2.0:1786177145.038466:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177145.038468:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.038470:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177145.038474:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177145/real 1786177145] req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177145.038482:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177145.038483:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177145.038485:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177145.038487:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177145.038489:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177145.038491:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177145.038494:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177145.038502:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038509:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177145.038510:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038512:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177145.038513:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177145.038517:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038524:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177145.038528:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038535:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177145.038537:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038539:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177145.038540:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038542:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177145.038544:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177145.038546:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177145.038549:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177145.038552:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177145.038556:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177145.038564:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581184:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177145.038568:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177145.038572:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038579:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177145.038580:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.038583:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177145.038585:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177145.038587:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177145.038590:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581312:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177145.038593:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177145.038595:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177145.038596:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177145.038598:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177145.038601:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786177145.038603:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177145.038606:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177145.038609:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177145.038612:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177145.038615:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235581312, portal 4 00000100:00000001:2.0:1786177145.038618:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177145.038620:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487994624 : -110371221556992 : ffff9b9e328bb100) 00000100:00000040:2.0:1786177145.038625:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038631:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177145.038634:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177145.038636:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177145.038639:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235581312, offset 0 00000400:00000200:2.0:1786177145.038642:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177145.038646:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177145.038650:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177145.038653:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177145.038656:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177145.038659:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177145.038661:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177145.038664:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426608:426608:256:4294967295] 192.168.202.132@tcp LPNI seq info [426570:426570:8:4294967295] 00000400:00000200:2.0:1786177145.038668:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177145.038675:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177145.038679:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177145.038683:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177145.038687:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177145.038690:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1786177145.038694:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177145.038696:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1786177145.038701:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177145.038703:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177145.038707:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177145.038710:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177145.038713:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177145.038715:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177145.038717:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177145.038721:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038734:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177145.038738:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038744:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.038746:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177145.038749:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177145.038752:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177145.038755:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177145.038757:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306491911 00000100:00000001:2.0:1786177145.038760:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306495744 00000100:00000001:2.0:1786177145.038762:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306491911 00000100:00000001:2.0:1786177145.038764:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306495744 00000100:00000001:2.0:1786177145.038767:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177145.038769:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.038771:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177145.038775:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177145/real 1786177145] req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177145.038782:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177145.038783:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177145.038785:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177145.038788:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177145.038790:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177145.038792:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177145.038796:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177145.038804:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038810:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177145.038812:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038814:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177145.038815:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177145.038819:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038826:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177145.038830:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038853:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177145.038856:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038857:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177145.038859:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038861:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177145.038863:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177145.038866:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177145.038869:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177145.038872:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177145.038877:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177145.038885:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581312:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177145.038889:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038892:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177145.038895:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8700 x1872930235580928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038902:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177145.038904:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177145.038906:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:2.0:1786177145.038909:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177145.038911:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177145.038912:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177145.038914:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177145.038916:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:2.0:1786177145.038919:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177145.038920:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177145.038921:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8700. 00000100:00000001:2.0:1786177145.038924:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177145.038925:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038927:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177145.038930:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9c00 x1872930235581056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038937:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177145.038938:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177145.038940:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177145.038942:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177145.038943:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177145.038945:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177145.038947:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177145.038948:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af000. 02000000:00000001:2.0:1786177145.038950:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177145.038952:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177145.038953:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9c00. 00000100:00000001:2.0:1786177145.038956:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177145.038957:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038959:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177145.038962:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328ba680 x1872930235581184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.038969:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177145.038971:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177145.038973:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786177145.038975:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177145.038976:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177145.038977:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177145.038979:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177145.038980:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:2.0:1786177145.038982:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177145.038984:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177145.038986:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328ba680. 00000100:00000001:2.0:1786177145.038989:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177145.038991:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.038992:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177145.038996:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb100 x1872930235581312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177145.039003:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177145.039005:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177145.039006:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786177145.039009:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177145.039010:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177145.039012:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177145.039014:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177145.039016:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af800. 02000000:00000001:2.0:1786177145.039018:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177145.039019:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177145.039021:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb100. 00000100:00000001:2.0:1786177145.039024:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177145.039025:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.039027:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177145.039031:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177145.039033:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177145.039034:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177145.039036:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.548419:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177145.548426:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177145.548428:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177145.548433:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177145.548436:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177145.548439:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177145.548442:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177145.548445:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177145.548447:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.932467:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177145.932470:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177145.932474:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.932475:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177145.932478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177145.932480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177145.932483:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177145.932484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.932489:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177145.932490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177145.932493:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177145.932494:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177145.932497:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177145.932500:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177145.932503:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177145.932559:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177145.932563:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177145.932566:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177146.572466:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177146.572475:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177146.572477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177146.572483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177146.572487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177146.572491:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177146.572494:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177146.572497:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177146.572501:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177146.956461:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177146.956466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177146.956470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177146.956471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177146.956474:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177146.956480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177146.956481:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177146.956484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177146.956487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177146.956491:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177146.956493:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177146.956495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177146.956499:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177146.956504:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177146.956508:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177146.956553:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177146.956556:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177146.956560:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177147.596445:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177147.596453:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177147.596455:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177147.596461:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177147.596465:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177147.596468:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177147.596470:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177147.596472:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177147.596474:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177147.980475:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177147.980480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177147.980482:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177147.980486:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177147.980487:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177147.980494:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177147.980496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177147.980498:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177147.980500:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177147.980502:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177147.980506:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177147.980506:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177147.980508:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177147.980509:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177147.980512:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177147.980586:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177147.980591:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177147.980595:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177148.620470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177148.620477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177148.620479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177148.620485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177148.620489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177148.620493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177148.620497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177148.620500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177148.620504:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177148.940444:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306495744, at 4306495808 00000100:00000040:1.0:1786177148.940445:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306495744, at 4306495808 00000100:00000010:1.0:1786177148.940462:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cae0. 00000100:00000010:0.0:1786177148.940467:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c4e0. 00000100:00000001:1.0:1786177149.004460:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177149.004464:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177149.004469:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177149.004469:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177149.004475:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177149.004477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177149.004478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177149.004480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177149.004483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177149.004484:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177149.004489:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177149.004494:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177149.004497:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177149.004500:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177149.004502:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177149.004557:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177149.004563:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177149.004567:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177149.644415:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177149.644421:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177149.644423:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177149.644427:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177149.644431:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177149.644433:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177149.644436:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177149.644437:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177149.644440:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.028427:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177150.028433:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177150.028435:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177150.028439:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177150.028440:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177150.028444:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.028446:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177150.028451:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177150.028451:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177150.028456:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177150.028456:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177150.028460:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177150.028461:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177150.028464:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.028465:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177150.028470:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177150.028473:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177150.028476:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177150.028478:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177150.028482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177150.028483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177150.028543:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177150.028548:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177150.028551:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177150.156452:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177150.156462:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177150.156465:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177150.156469:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177150.156473:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177150.156480:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11824 00000400:00000200:1.0:1786177150.156488:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177150.156495:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177150.156498:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177150.156501:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156504:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177150.156507:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156510:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177150.156513:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177150.156515:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177150.156518:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177150.156521:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177150.156523:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177150.156526:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177150.156529:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156531:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177150.156533:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156535:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177150.156538:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177150.156541:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156544:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177150.156547:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177150.156550:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177150.156553:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.156555:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177150.156557:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177150.156566:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb480. 00000020:00000040:1.0:1786177150.156569:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177150.156573:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177150.156574:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.156577:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177150.156580:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177150.156583:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177150.156590:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261adc00. 00000100:00000001:1.0:1786177150.156596:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177150.156605:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177150.156615:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177150.156620:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177150.156625:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b192a357] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177150.156645:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177150.156648:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177150.156651:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177150.156653:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177150.156654:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177150.156657:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177150.156659:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11824 00000400:00000200:1.0:1786177150.156663:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177150.156666:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177150.156668:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177150.156680:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156683:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177150.156685:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177150.156685:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177150.156690:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177150.156691:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177150.156694:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1786177150.156695:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177150.156698:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177150.156700:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:2.0:1786177150.156703:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177150.156704:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177150.156708:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177150.156713:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177150.156720:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177150.156722:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177150.156724:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177150.156725:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177150.156728:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156731:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:2.0:1786177150.156731:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177150.156735:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1786177150.156736:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:1.0:1786177150.156742:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:2.0:1786177150.156742:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.156748:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177150.156749:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235581440:192.168.202.132@tcp:250:kworker.0 02000000:00000001:1.0:1786177150.156754:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177150.156756:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177150.156760:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177150.156761:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177150.156764:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177150.156765:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177150.156769:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1786177150.156770:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177150.156775:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000010:2.0:1786177150.156775:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786177150.156779:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1786177150.156780:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.156784:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177150.156799:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9f80. 00000400:00000200:2.0:1786177150.156800:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:1.0:1786177150.156804:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000010:2.0:1786177150.156807:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 02000000:00000001:1.0:1786177150.156808:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177150.156811:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.156814:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177150.156816:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1786177150.156816:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 02000000:00000001:1.0:1786177150.156818:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177150.156824:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af400. 00000100:00000200:2.0:1786177150.156825:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235581440, portal 25 00000100:00000001:1.0:1786177150.156829:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.156830:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177150.156833:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487995520 : -110371221556096 : ffff9b9e328bb480) 00000100:00100000:1.0:1786177150.156839:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1786177150.156842:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177150.156850:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177150.156853:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177150.156860:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4f3e5a2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.156871:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177150.156877:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156878:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177150.156882:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00080000:1.0:1786177150.156883:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:2.0:1786177150.156887:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235581440, offset 0 00000100:00080000:1.0:1786177150.156888:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177150.156892:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177150.156894:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177150.156899:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177150.156899:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:1.0:1786177150.156905:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11824 00000400:00000200:2.0:1786177150.156909:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177150.156913:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177150.156920:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:2.0:1786177150.156920:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00080000:1.0:1786177150.156925:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177150.156929:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1786177150.156931:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156934:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177150.156936:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:1.0:1786177150.156939:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156942:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177150.156944:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1786177150.156946:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177150.156949:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000400:00000200:2.0:1786177150.156950:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000001:1.0:1786177150.156953:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:2.0:1786177150.156958:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426609:426609:256:4294967295] 192.168.202.132@tcp LPNI seq info [426571:426571:8:4294967295] 00000400:00000200:2.0:1786177150.156962:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:1.0:1786177150.156964:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177150.156968:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177150.156974:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786177150.156979:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000040:1.0:1786177150.156981:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156985:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177150.156989:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:2.0:1786177150.156991:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177150.156992:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177150.156996:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000010:2.0:1786177150.156998:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00080000:1.0:1786177150.156999:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177150.157004:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177150.157007:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.0:1786177150.157009:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:1.0:1786177150.157012:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1786177150.157014:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 02000000:00000001:1.0:1786177150.157016:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.157020:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:2.1:1786177150.157023:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177150.157027:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000008:00100000:1.0:1786177150.157030:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1786177150.157034:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000008:00000001:1.0:1786177150.157036:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1786177150.157038:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000100:00000001:1.0:1786177150.157040:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177150.157044:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb800. 00000400:00000200:2.0:1786177150.157045:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:1.0:1786177150.157048:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177150.157054:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1786177150.157054:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1786177150.157057:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.157060:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1786177150.157061:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 02000000:00000001:1.0:1786177150.157062:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177150.157065:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177150.157066:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 02000000:00000010:1.0:1786177150.157071:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac000. 00000100:00000001:2.0:1786177150.157072:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1786177150.157078:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177150.157079:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177150.157087:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177150.157093:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1786177150.157099:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1786177150.157100:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177150.157104:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.157114:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177150.157115:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008148c2a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.157119:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177150.157125:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.157126:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177150.157131:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1786177150.157132:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177150.157137:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177150.157137:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306500864 00000100:00000010:2.0:1786177150.157142:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cb40. 00000100:00000001:1.0:1786177150.157147:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177150.157149:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177150.157149:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177150.157155:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177150.157160:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11824 00000100:00000001:2.0:1786177150.157160:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306500864 00000100:00000010:2.0:1786177150.157165:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c660. 00000400:00000200:1.0:1786177150.157169:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177150.157170:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000400:00000200:1.0:1786177150.157175:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177150.157175:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1786177150.157180:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177150.157181:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.157186:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1786177150.157187:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.157191:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1786177150.157193:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177150/real 1786177150] req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177150.157195:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.157201:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177150.157205:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177150.157207:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000400:00000001:2.0:1786177150.157209:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177150.157212:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177150.157213:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177150.157216:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000020:00000001:1.0:1786177150.157218:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:2.0:1786177150.157219:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000020:00000040:1.0:1786177150.157222:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:2.0:1786177150.157223:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1786177150.157228:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000200:2.0:1786177150.157230:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177150.157233:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:1.0:1786177150.157242:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.157246:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1786177150.157247:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177150.157251:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.157257:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177150.157262:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:2.0:1786177150.157263:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177150.157267:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1786177150.157268:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177150.157270:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177150.157271:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.157273:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177150.157275:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177150.157279:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1786177150.157281:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177150.157283:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.157287:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177150.157291:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177150.157296:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.157296:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177150.157300:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177150.157302:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1786177150.157306:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9180. 00000020:00000040:1.0:1786177150.157311:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:2.0:1786177150.157314:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1786177150.157316:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177150.157318:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177150.157319:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.157321:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177150.157323:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1786177150.157324:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177150.157326:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1786177150.157328:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:1.0:1786177150.157330:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.157331:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:1.0:1786177150.157335:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad000. 00000100:00080000:2.0:1786177150.157336:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1786177150.157341:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177150.157343:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177150.157347:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1786177150.157350:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1786177150.157355:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177150.157381:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177150.157384:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177150.157390:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d01dc3b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177150.157404:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177150.157409:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1786177150.157414:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235581440:192.168.202.132@tcp:250:kworker.0 00000100:00080000:1.0:1786177150.157415:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177150.157418:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177150.157421:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177150.157422:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177150.157424:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786177150.157426:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1786177150.157429:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11824 00000100:00000040:2.0:1786177150.157433:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb480 x1872930235581440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177150.157438:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177150.157446:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177150.157448:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:1.0:1786177150.157450:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1786177150.157452:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:1.0:1786177150.157455:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000010:2.0:1786177150.157456:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab400. 00000100:00000001:1.0:1786177150.157460:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1786177150.157462:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177150.157465:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000040:1.0:1786177150.157466:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000020:00000040:2.0:1786177150.157468:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000001:1.0:1786177150.157470:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177150.157472:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:1.0:1786177150.157474:0:40394:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000010:2.0:1786177150.157476:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261adc00. 00000020:00000040:1.0:1786177150.157477:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177150.157481:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:2.0:1786177150.157482:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000040:1.0:1786177150.157483:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 02000000:00000001:2.0:1786177150.157485:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:1.0:1786177150.157488:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000010:2.0:1786177150.157489:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb480. 00000020:00000001:1.0:1786177150.157494:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177150.157495:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177150.157498:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177150.157499:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177150.157501:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177150.157503:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177150.157507:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177150.157509:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177150.157509:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177150.157512:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177150.157514:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:2.0:1786177150.157516:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177150.157520:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1786177150.157522:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177150.157523:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177150.157528:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177150.157530:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177150.157531:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177150.157535:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177150.157540:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177150.157544:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177150.157545:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000008:00100000:1.0:1786177150.157547:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1786177150.157548:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1786177150.157551:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177150.157553:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1786177150.157555:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177150.157556:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1786177150.157560:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8a80. 02000000:00000001:2.0:1786177150.157561:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177150.157566:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00100000:2.0:1786177150.157568:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581568:192.168.202.132@tcp:38:kworker.0 02000000:00000001:1.0:1786177150.157571:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177150.157573:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177150.157574:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177150.157576:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1786177150.157578:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177150.157580:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177150.157582:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177150.157583:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1786177150.157586:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177150.157588:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786177150.157591:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177150.157593:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac800. 00000400:00000200:2.0:1786177150.157598:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177150.157599:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177150.157603:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00100000:1.0:1786177150.157607:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1786177150.157609:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177150.157614:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235581568, portal 10 00000100:00000001:2.0:1786177150.157618:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177150.157620:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487990144 : -110371221561472 : ffff9b9e328b9f80) 00000100:00000001:1.0:1786177150.157630:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1786177150.157631:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177150.157634:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177150.157642:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002bb61190] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.157643:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177150.157649:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177150.157652:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177150.157656:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235581568, offset 0 00000100:00000001:1.0:1786177150.157660:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177150.157664:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000040:1.0:1786177150.157668:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11834) 00000400:00000200:2.0:1786177150.157671:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177150.157676:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177150.157680:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177150.157683:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177150.157685:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177150.157688:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177150.157692:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426610:426610:256:4294967295] 192.168.202.132@tcp LPNI seq info [426572:426572:8:4294967295] 00000400:00000200:2.0:1786177150.157696:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177150.157703:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177150.157707:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177150.157710:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177150.157714:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177150.157717:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786177150.157721:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177150.157723:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1786177150.157727:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177150.157729:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177150.157731:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177150.157735:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177150.157737:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177150.157739:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177150.157741:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177150.157745:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.157752:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177150.157755:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.157761:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.157763:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177150.157765:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177150.157769:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177150.157772:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177150.157773:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306497028 00000100:00000001:2.0:1786177150.157776:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306500864 00000100:00000001:2.0:1786177150.157778:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306497028 00000100:00000001:2.0:1786177150.157780:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306500864 00000100:00000001:2.0:1786177150.157782:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177150.157783:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.157797:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177150.157801:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177150/real 1786177150] req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177150.157814:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177150.157816:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177150.157817:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177150.157820:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177150.157821:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177150.157823:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177150.157826:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177150.157834:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.157841:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177150.157842:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.157843:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177150.157844:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177150.157848:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.157855:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177150.157859:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.157865:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177150.157867:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.157869:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177150.157870:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.157871:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177150.157873:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177150.157875:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177150.157878:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177150.157881:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177150.157885:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177150.157892:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581568:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177150.157896:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177150.157899:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.157905:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177150.157906:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177150.157908:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177150.157910:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177150.157912:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177150.157921:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581696:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177150.157924:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177150.157926:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177150.157930:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177150.157932:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177150.157935:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786177150.157937:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177150.157940:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177150.157942:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177150.157944:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177150.157947:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235581696, portal 10 00000100:00000001:2.0:1786177150.157949:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177150.157950:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487996416 : -110371221555200 : ffff9b9e328bb800) 00000100:00000040:2.0:1786177150.157955:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.157962:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177150.157964:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177150.157966:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177150.157968:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235581696, offset 0 00000400:00000200:2.0:1786177150.157972:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177150.157976:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177150.157979:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177150.157981:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177150.157984:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177150.157986:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177150.157988:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177150.157991:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426611:426611:256:4294967295] 192.168.202.132@tcp LPNI seq info [426573:426573:8:4294967295] 00000400:00000200:2.0:1786177150.157995:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177150.158001:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177150.158006:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177150.158008:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177150.158012:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177150.158015:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786177150.158019:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177150.158022:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1786177150.158025:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177150.158027:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177150.158029:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177150.158032:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177150.158035:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177150.158037:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177150.158039:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177150.158042:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158049:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177150.158052:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158057:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.158059:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177150.158061:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177150.158064:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177150.158066:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177150.158069:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306497028 00000100:00000001:2.0:1786177150.158071:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306500864 00000100:00000001:2.0:1786177150.158074:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306497028 00000100:00000001:2.0:1786177150.158076:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306500864 00000100:00000001:2.0:1786177150.158078:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177150.158079:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.158081:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177150.158084:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177150/real 1786177150] req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177150.158091:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177150.158092:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177150.158094:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177150.158095:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177150.158097:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177150.158098:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177150.158102:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177150.158110:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158115:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177150.158117:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158118:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177150.158120:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177150.158123:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158130:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177150.158134:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158140:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177150.158142:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158144:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177150.158145:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158147:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177150.158148:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177150.158151:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177150.158153:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177150.158155:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177150.158158:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177150.158166:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581696:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177150.158169:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177150.158172:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158178:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177150.158179:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177150.158181:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177150.158183:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177150.158185:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177150.158187:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581824:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177150.158190:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177150.158191:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177150.158193:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177150.158195:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177150.158197:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177150.158199:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177150.158202:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177150.158204:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177150.158207:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177150.158210:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235581824, portal 4 00000100:00000001:2.0:1786177150.158212:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177150.158213:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487986560 : -110371221565056 : ffff9b9e328b9180) 00000100:00000040:2.0:1786177150.158218:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158224:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177150.158226:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177150.158227:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177150.158230:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235581824, offset 0 00000400:00000200:2.0:1786177150.158233:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177150.158236:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177150.158239:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177150.158241:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177150.158243:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177150.158246:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177150.158248:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177150.158251:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426612:426612:256:4294967295] 192.168.202.132@tcp LPNI seq info [426574:426574:8:4294967295] 00000400:00000200:2.0:1786177150.158255:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177150.158261:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177150.158265:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177150.158267:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177150.158270:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177150.158273:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786177150.158277:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177150.158279:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1786177150.158282:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177150.158284:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177150.158287:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177150.158290:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177150.158293:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177150.158294:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177150.158296:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177150.158299:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158313:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177150.158316:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158323:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.158324:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177150.158326:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177150.158329:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177150.158332:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177150.158334:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306497028 00000100:00000001:2.0:1786177150.158336:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306500864 00000100:00000001:2.0:1786177150.158338:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306497028 00000100:00000001:2.0:1786177150.158340:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306500864 00000100:00000001:2.0:1786177150.158341:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177150.158343:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.158345:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177150.158348:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177150/real 1786177150] req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177150.158355:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177150.158372:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177150.158373:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177150.158376:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177150.158377:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177150.158379:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177150.158382:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177150.158391:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158396:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177150.158398:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158400:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177150.158401:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177150.158404:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158411:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177150.158414:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158421:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177150.158423:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158425:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177150.158426:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158427:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177150.158428:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177150.158430:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177150.158433:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177150.158435:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177150.158439:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177150.158446:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581824:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177150.158450:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158452:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177150.158454:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9f80 x1872930235581568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158460:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177150.158462:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177150.158463:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:2.0:1786177150.158465:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177150.158467:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177150.158468:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177150.158470:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177150.158471:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af400. 02000000:00000001:2.0:1786177150.158474:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177150.158475:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177150.158476:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9f80. 00000100:00000001:2.0:1786177150.158480:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177150.158481:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158483:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177150.158486:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb800 x1872930235581696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158492:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177150.158493:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177150.158495:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:2.0:1786177150.158497:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177150.158498:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177150.158499:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177150.158500:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177150.158502:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:2.0:1786177150.158503:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177150.158504:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177150.158506:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb800. 00000100:00000001:2.0:1786177150.158508:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177150.158509:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158511:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177150.158514:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9180 x1872930235581824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158520:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177150.158521:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177150.158522:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177150.158524:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177150.158525:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177150.158526:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177150.158527:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177150.158529:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:2.0:1786177150.158530:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177150.158532:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177150.158533:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9180. 00000100:00000001:2.0:1786177150.158535:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177150.158536:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158538:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158541:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177150.158542:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.158543:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177150.158545:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177150.158546:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177150.158550:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158557:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177150.158558:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177150.158560:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177150.158561:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177150.158563:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177150.158566:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581952:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177150.158569:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177150.158571:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177150.158572:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177150.158574:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177150.158576:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177150.158578:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177150.158581:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177150.158583:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177150.158586:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177150.158588:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235581952, portal 4 00000100:00000001:2.0:1786177150.158590:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177150.158591:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487984768 : -110371221566848 : ffff9b9e328b8a80) 00000100:00000040:2.0:1786177150.158596:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158601:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177150.158604:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177150.158606:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177150.158608:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235581952, offset 0 00000400:00000200:2.0:1786177150.158612:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177150.158615:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177150.158618:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177150.158621:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177150.158623:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177150.158625:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177150.158627:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177150.158630:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426613:426613:256:4294967295] 192.168.202.132@tcp LPNI seq info [426575:426575:8:4294967295] 00000400:00000200:2.0:1786177150.158634:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177150.158640:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177150.158644:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177150.158646:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177150.158650:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177150.158652:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1786177150.158656:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177150.158659:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1786177150.158662:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177150.158664:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177150.158666:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177150.158669:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177150.158672:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177150.158673:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177150.158675:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177150.158679:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158686:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177150.158688:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177175 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158695:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.158697:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177150.158699:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177150.158702:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177150.158705:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177150.158707:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306497031 00000100:00000001:2.0:1786177150.158709:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306500864 00000100:00000001:2.0:1786177150.158711:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306497031 00000100:00000001:2.0:1786177150.158713:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306500864 00000100:00000001:2.0:1786177150.158715:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177150.158717:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.158718:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177150.158722:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177150/real 1786177150] req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177150.158729:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177150.158730:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177150.158731:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177150.158733:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177150.158735:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177150.158737:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177150.158740:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177150.158747:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158753:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177150.158755:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158756:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177150.158758:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177150.158761:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158767:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177150.158771:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158778:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177150.158780:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158782:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177150.158783:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158784:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177150.158796:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177150.158799:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177150.158802:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177150.158804:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177150.158808:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177150.158816:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235581952:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177150.158819:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158821:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177150.158824:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8a80 x1872930235581952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177175 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177150.158830:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177150.158832:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177150.158833:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:2.0:1786177150.158836:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177150.158837:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177150.158838:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177150.158840:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177150.158842:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:2.0:1786177150.158844:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177150.158845:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177150.158846:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8a80. 00000100:00000001:2.0:1786177150.158849:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177150.158850:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158852:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177150.158854:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177150.158856:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177150.158858:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177150.158859:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177150.668444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177150.668451:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177150.668454:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177150.668460:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177150.668464:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177150.668467:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177150.668470:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177150.668472:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177150.668475:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177151.052477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177151.052481:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177151.052486:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177151.052487:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177151.052491:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177151.052494:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177151.052496:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177151.052498:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177151.052499:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177151.052503:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177151.052508:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177151.052513:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177151.052517:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177151.052521:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177151.052525:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177151.052565:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177151.052569:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177151.052573:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177151.692430:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177151.692436:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177151.692438:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177151.692443:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177151.692446:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177151.692449:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177151.692450:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177151.692453:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177151.692454:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177152.076445:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177152.076448:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177152.076452:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177152.076454:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177152.076456:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177152.076458:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177152.076458:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177152.076465:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177152.076466:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177152.076466:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177152.076472:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177152.076475:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177152.076477:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177152.076479:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177152.076481:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177152.076530:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177152.076535:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177152.076538:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177152.717331:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177152.717339:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177152.717341:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177152.717346:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177152.717349:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177152.717353:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177152.717377:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177152.717380:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177152.717383:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177153.100434:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177153.100438:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177153.100440:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177153.100443:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177153.100443:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177153.100447:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1786177153.100448:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177153.100450:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177153.100452:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177153.100454:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177153.100456:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177153.100457:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177153.100460:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177153.100460:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177153.100463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177153.100464:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177153.100468:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177153.100469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177153.100471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177153.100479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177153.100481:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177153.100482:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177153.100485:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177153.100487:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177153.100488:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177153.100489:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177153.100490:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177153.100530:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177153.100533:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177153.100535:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177153.740537:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177153.740547:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177153.740549:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177153.740553:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177153.740557:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177153.740561:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177153.740818:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177153.740822:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177153.740826:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177154.060431:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306500864, at 4306500928 00000100:00000040:1.0:1786177154.060431:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306500864, at 4306500928 00000100:00000010:1.0:1786177154.060446:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c660. 00000100:00000010:0.0:1786177154.060449:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cb40. 00000100:00000001:1.0:1786177154.124445:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177154.124450:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177154.124451:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177154.124454:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177154.124454:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177154.124459:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177154.124460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177154.124463:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177154.124463:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177154.124466:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177154.124468:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177154.124469:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177154.124470:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177154.124473:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177154.124477:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177154.124519:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177154.124522:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177154.124524:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177154.764711:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177154.764721:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177154.764724:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177154.764731:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177154.764736:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177154.764740:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177154.764759:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177154.764762:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177154.764765:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177155.148489:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177155.148489:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177155.148495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.148499:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177155.148501:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177155.148505:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.148505:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177155.148519:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177155.148521:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177155.148522:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.148535:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177155.148541:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.148545:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177155.148549:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.148554:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177155.148587:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177155.148590:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177155.148592:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177155.276508:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177155.276522:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177155.276525:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177155.276530:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177155.276536:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177155.276544:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11829 00000400:00000200:2.0:1786177155.276556:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177155.276563:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177155.276566:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177155.276569:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177155.276572:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177155.276574:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177155.276576:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177155.276580:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177155.276581:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177155.276584:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177155.276587:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177155.276588:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177155.276590:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177155.276593:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177155.276594:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177155.276596:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177155.276598:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177155.276600:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177155.276603:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177155.276605:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177155.276607:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177155.276611:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177155.276613:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177155.276615:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177155.276617:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177155.276628:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000020:00000040:2.0:1786177155.276631:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177155.276636:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177155.276637:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177155.276647:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177155.276651:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177155.276655:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177155.276663:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786177155.276669:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177155.276678:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177155.276688:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177155.276692:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177155.276696:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177155.276714:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177155.276716:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177155.276719:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177155.276720:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177155.276722:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177155.276724:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177155.276726:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11829 00000400:00000200:2.0:1786177155.276730:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177155.276733:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177155.276735:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177155.276749:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177155.276750:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177155.276751:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177155.276755:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177155.276756:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177155.276757:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177155.276760:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1786177155.276761:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177155.276764:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177155.276765:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177155.276768:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00000040:1.0:1786177155.276770:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177155.276770:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177155.276771:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177155.276781:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1786177155.276782:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177155.276785:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177155.276785:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177155.276789:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177155.276791:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1786177155.276792:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:1.0:1786177155.276796:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1786177155.276796:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 02000000:00000001:1.0:1786177155.276803:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177155.276803:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177155.276808:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177155.276811:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235582080:192.168.202.132@tcp:250:kworker.0 02000000:00000001:2.0:1786177155.276812:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177155.276817:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177155.276819:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177155.276821:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177155.276822:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177155.276824:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177155.276825:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1786177155.276829:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1786177155.276832:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177155.276835:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177155.276836:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae400. 00000100:00000001:2.0:1786177155.276837:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177155.276840:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177155.276843:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 00000020:00000040:2.0:1786177155.276846:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177155.276849:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177155.276850:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1786177155.276854:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 02000000:00000001:2.0:1786177155.276854:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177155.276856:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177155.276859:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177155.276860:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177155.276862:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 02000000:00000010:2.0:1786177155.276865:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000200:1.0:1786177155.276870:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235582080, portal 25 00000100:00000001:2.0:1786177155.276870:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.276874:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00100000:2.0:1786177155.276875:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177155.276877:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469625472 : -110371239926144 : ffff9b9e31736680) 00000100:00000001:2.0:1786177155.276884:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177155.276887:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177155.276891:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177155.276895:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177155.276904:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177155.276909:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177155.276910:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1786177155.276914:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:1.0:1786177155.276916:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177155.276917:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177155.276920:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1786177155.276921:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000001:2.0:1786177155.276923:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:1.0:1786177155.276925:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235582080, offset 0 00000100:00080000:2.0:1786177155.276928:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11829 00000400:00000200:2.0:1786177155.276935:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177155.276939:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177155.276942:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177155.276945:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177155.276947:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177155.276949:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177155.276952:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177155.276953:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177155.276956:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177155.276958:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177155.276959:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177155.276960:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1786177155.276962:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000040:2.0:1786177155.276968:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:1.0:1786177155.276969:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1786177155.276973:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177155.276977:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000001:2.0:1786177155.276977:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177155.276982:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1786177155.276987:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177155.276988:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:2.0:1786177155.276992:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177155.276997:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177155.276998:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426614:426614:256:4294967295] 192.168.202.132@tcp LPNI seq info [426576:426576:8:4294967295] 00000100:00000001:2.0:1786177155.277000:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177155.277004:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:1.0:1786177155.277005:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:2.0:1786177155.277008:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177155.277010:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177155.277011:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177155.277014:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177155.277016:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177155.277018:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177155.277020:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177155.277025:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177155.277026:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177155.277027:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177155.277035:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 00000800:00000200:1.0:1786177155.277037:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000020:00000040:2.0:1786177155.277038:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177155.277042:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177155.277043:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177155.277044:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000100:00000001:2.0:1786177155.277045:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177155.277047:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177155.277049:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177155.277053:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab400. 00000800:00000200:1.0:1786177155.277056:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177155.277059:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177155.277062:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00100000:2.0:1786177155.277069:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1786177155.277069:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177155.277075:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1786177155.277084:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177155.277088:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000100:00000001:2.0:1786177155.277089:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177155.277093:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177155.277094:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177155.277102:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1786177155.277102:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177155.277108:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177155.277112:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177155.277118:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1786177155.277119:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177155.277124:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177155.277125:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177155.277136:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177155.277140:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177155.277141:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1786177155.277142:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1786177155.277146:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177155.277149:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177155.277155:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177155.277156:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11829 00000100:00000001:1.0:1786177155.277157:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1786177155.277164:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177155.277169:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177155.277171:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177155.277172:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177155.277178:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177155.277181:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177155.277182:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306505984 00000100:00000010:1.0:1786177155.277187:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62540. 00000100:00000001:1.0:1786177155.277194:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177155.277194:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177155.277200:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177155.277204:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177155.277205:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306505984 00000020:00000001:2.0:1786177155.277206:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177155.277210:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000010:1.0:1786177155.277211:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62840. 00000020:00000001:2.0:1786177155.277212:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177155.277216:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:1.0:1786177155.277217:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1786177155.277220:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177155.277222:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1786177155.277223:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177155.277225:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177155.277228:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177155.277230:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1786177155.277231:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177155.277234:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00100000:1.0:1786177155.277238:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177155/real 1786177155] req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177155.277242:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000001:1.0:1786177155.277246:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:2.0:1786177155.277246:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000400:00000001:1.0:1786177155.277249:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1786177155.277252:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000010:1.0:1786177155.277253:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000100:00000001:2.0:1786177155.277258:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177155.277259:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177155.277263:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 02000000:00000001:2.0:1786177155.277264:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177155.277268:0:2392:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1786177155.277269:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177155.277273:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177155.277274:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177155.277275:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177155.277280:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177155.277284:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177155.277285:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177155.277287:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177155.277292:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000100:00000001:1.0:1786177155.277294:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:2.0:1786177155.277295:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177155.277297:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1786177155.277298:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177155.277300:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1786177155.277301:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.277304:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177155.277304:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177155.277307:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177155.277309:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177155.277310:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1786177155.277313:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:2.0:1786177155.277318:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.277319:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177155.277326:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1786177155.277327:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.277336:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177155.277341:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.277345:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177155.277348:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177155.277348:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177155.277352:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.277353:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177155.277384:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177155.277387:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1786177155.277391:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177155.277392:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177155.277399:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1786177155.277404:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177155.277410:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177155.277410:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177155.277420:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177155.277425:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177155.277429:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1786177155.277432:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235582080:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177155.277433:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177155.277440:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11829 00000100:00000001:1.0:1786177155.277442:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1786177155.277448:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177155.277449:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1786177155.277455:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000040:1.0:1786177155.277457:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736680 x1872930235582080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177155.277461:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177155.277469:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177155.277472:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1786177155.277474:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177155.277478:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:2.0:1786177155.277479:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000010:1.0:1786177155.277484:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae400. 00000100:00000001:2.0:1786177155.277484:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177155.277490:0:40394:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1786177155.277491:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000040:2.0:1786177155.277492:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177155.277495:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000001:2.0:1786177155.277497:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177155.277500:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000040:2.0:1786177155.277502:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:1.0:1786177155.277506:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000001:2.0:1786177155.277507:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 02000000:00000010:1.0:1786177155.277510:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9400. 00000020:00000001:2.0:1786177155.277512:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 02000000:00000001:1.0:1786177155.277518:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:2.0:1786177155.277518:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177155.277522:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177155.277523:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177155.277528:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736680. 00000100:00000040:2.0:1786177155.277528:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177155.277535:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177155.277535:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177155.277538:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:1.0:1786177155.277539:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1786177155.277545:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177155.277546:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177155.277549:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177155.277554:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177155.277557:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1786177155.277557:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177155.277562:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177155.277562:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.277569:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177155.277569:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177155.277572:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177155.277574:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000008:00000001:2.0:1786177155.277577:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.277579:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1786177155.277582:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177155.277588:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1786177155.277589:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000020:00000040:2.0:1786177155.277595:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177155.277602:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177155.277604:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.277607:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177155.277608:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177155.277611:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177155.277612:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177155.277615:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177155.277618:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:2.0:1786177155.277622:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 02000000:00000001:1.0:1786177155.277624:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177155.277627:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177155.277630:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177155.277638:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582208:192.168.202.132@tcp:38:kworker.0 00000100:00100000:2.0:1786177155.277638:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177155.277648:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177155.277652:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177155.277656:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177155.277661:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177155.277664:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177155.277670:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1786177155.277671:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177155.277676:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177155.277682:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177155.277690:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177155.277691:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177155.277696:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000100:00000040:2.0:1786177155.277700:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11839) 00000400:00000010:1.0:1786177155.277702:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177155.277708:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235582208, portal 10 00000100:00000001:1.0:1786177155.277710:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177155.277712:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469626368 : -110371239925248 : ffff9b9e31736a00) 00000100:00000040:1.0:1786177155.277721:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.277729:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177155.277732:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177155.277735:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177155.277755:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235582208, offset 0 00000400:00000200:1.0:1786177155.277760:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177155.277764:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177155.277770:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177155.277773:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177155.277776:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177155.277779:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177155.277782:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177155.277785:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426615:426615:256:4294967295] 192.168.202.132@tcp LPNI seq info [426577:426577:8:4294967295] 00000400:00000200:1.0:1786177155.277789:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177155.277796:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177155.277802:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177155.277806:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177155.277810:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177155.277813:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786177155.277817:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177155.277820:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1786177155.277824:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177155.277827:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177155.277830:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177155.277834:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177155.277838:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177155.277840:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177155.277843:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177155.277847:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.277855:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177155.277859:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.277866:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.277868:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177155.277872:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177155.277876:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177155.277880:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177155.277882:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306502148 00000100:00000001:1.0:1786177155.277886:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306505984 00000100:00000001:1.0:1786177155.277889:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306502148 00000100:00000001:1.0:1786177155.277891:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306505984 00000100:00000001:1.0:1786177155.277893:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177155.277896:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.277898:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177155.277902:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177155/real 1786177155] req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177155.277911:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177155.277913:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177155.277915:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177155.277917:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177155.277920:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177155.277922:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177155.277926:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177155.277934:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.277942:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177155.277943:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.277944:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177155.277946:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177155.277948:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.277956:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177155.277960:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.277968:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177155.277971:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.277974:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177155.277975:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.277977:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177155.277980:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177155.277983:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177155.277987:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177155.277990:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177155.277994:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177155.278003:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582208:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177155.278007:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177155.278011:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278016:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177155.278017:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177155.278019:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177155.278021:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177155.278022:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177155.278024:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582336:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177155.278027:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177155.278028:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177155.278029:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177155.278031:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177155.278033:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177155.278035:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177155.278037:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177155.278039:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177155.278041:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177155.278044:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235582336, portal 10 00000100:00000001:1.0:1786177155.278045:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177155.278046:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469615616 : -110371239936000 : ffff9b9e31734000) 00000100:00000040:1.0:1786177155.278051:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278056:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177155.278058:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177155.278059:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177155.278061:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235582336, offset 0 00000400:00000200:1.0:1786177155.278063:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177155.278066:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177155.278069:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177155.278071:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177155.278074:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177155.278076:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177155.278079:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177155.278082:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426616:426616:256:4294967295] 192.168.202.132@tcp LPNI seq info [426578:426578:8:4294967295] 00000400:00000200:1.0:1786177155.278087:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177155.278094:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177155.278099:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177155.278103:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177155.278107:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177155.278109:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786177155.278113:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177155.278116:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1786177155.278120:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177155.278122:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177155.278125:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177155.278128:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177155.278131:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177155.278133:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177155.278136:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177155.278140:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278154:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177155.278157:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278165:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.278166:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177155.278170:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177155.278174:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177155.278178:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177155.278180:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306502148 00000100:00000001:1.0:1786177155.278183:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306505984 00000100:00000001:1.0:1786177155.278185:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306502148 00000100:00000001:1.0:1786177155.278188:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306505984 00000100:00000001:1.0:1786177155.278190:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177155.278192:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.278195:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177155.278200:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177155/real 1786177155] req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177155.278209:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177155.278210:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177155.278213:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177155.278215:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177155.278217:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177155.278219:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177155.278223:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177155.278232:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278239:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177155.278240:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278243:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177155.278244:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177155.278248:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278254:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177155.278258:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278264:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177155.278266:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278267:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177155.278268:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278270:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177155.278272:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177155.278274:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177155.278277:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177155.278279:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177155.278283:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177155.278291:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582336:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177155.278296:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177155.278299:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278307:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177155.278308:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177155.278311:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177155.278313:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177155.278315:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177155.278319:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582464:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177155.278322:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177155.278324:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177155.278326:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177155.278328:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177155.278331:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177155.278333:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177155.278337:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177155.278340:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177155.278344:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177155.278347:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235582464, portal 4 00000100:00000001:1.0:1786177155.278349:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177155.278350:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469624576 : -110371239927040 : ffff9b9e31736300) 00000100:00000040:1.0:1786177155.278378:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278387:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177155.278389:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177155.278392:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177155.278395:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235582464, offset 0 00000400:00000200:1.0:1786177155.278399:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177155.278403:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177155.278407:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177155.278410:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177155.278413:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177155.278415:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177155.278418:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177155.278421:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426617:426617:256:4294967295] 192.168.202.132@tcp LPNI seq info [426579:426579:8:4294967295] 00000400:00000200:1.0:1786177155.278426:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177155.278433:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177155.278437:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177155.278441:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177155.278445:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177155.278448:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786177155.278453:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177155.278456:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1786177155.278460:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177155.278462:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177155.278465:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177155.278469:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177155.278472:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177155.278474:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177155.278476:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177155.278480:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278488:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177155.278491:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278497:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.278499:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177155.278502:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177155.278504:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177155.278507:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177155.278510:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306502149 00000100:00000001:1.0:1786177155.278512:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306505984 00000100:00000001:1.0:1786177155.278514:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306502149 00000100:00000001:1.0:1786177155.278517:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306505984 00000100:00000001:1.0:1786177155.278519:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177155.278521:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.278524:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177155.278528:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177155/real 1786177155] req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177155.278536:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177155.278539:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177155.278540:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177155.278543:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177155.278544:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177155.278547:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177155.278550:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177155.278557:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278563:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177155.278565:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278567:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177155.278569:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177155.278572:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278580:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177155.278584:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278591:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177155.278594:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278596:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177155.278597:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278599:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177155.278601:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177155.278604:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177155.278608:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177155.278611:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177155.278615:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177155.278631:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582464:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177155.278635:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278637:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177155.278641:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736a00 x1872930235582208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278647:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177155.278649:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177155.278652:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177155.278655:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177155.278657:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177155.278658:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177155.278661:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177155.278663:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:1.0:1786177155.278665:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177155.278667:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177155.278669:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736a00. 00000100:00000001:1.0:1786177155.278671:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177155.278672:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278674:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177155.278678:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734000 x1872930235582336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278685:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177155.278686:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177155.278688:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177155.278691:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177155.278693:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177155.278694:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177155.278696:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177155.278698:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:1.0:1786177155.278700:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177155.278701:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177155.278703:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734000. 00000100:00000001:1.0:1786177155.278706:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177155.278708:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278710:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177155.278713:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736300 x1872930235582464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278720:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177155.278722:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177155.278723:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177155.278726:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177155.278727:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177155.278729:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177155.278731:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177155.278732:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:1.0:1786177155.278735:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177155.278751:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177155.278752:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736300. 00000100:00000001:1.0:1786177155.278755:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177155.278757:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278759:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278763:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177155.278764:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.278767:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177155.278768:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177155.278770:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177155.278774:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278782:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177155.278784:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177155.278786:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177155.278788:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177155.278790:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177155.278793:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582592:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177155.278797:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177155.278799:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177155.278801:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177155.278803:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177155.278806:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177155.278808:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177155.278812:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177155.278814:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177155.278818:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177155.278821:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235582592, portal 4 00000100:00000001:1.0:1786177155.278823:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177155.278824:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469616512 : -110371239935104 : ffff9b9e31734380) 00000100:00000040:1.0:1786177155.278831:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278836:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177155.278837:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177155.278839:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177155.278841:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235582592, offset 0 00000400:00000200:1.0:1786177155.278844:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177155.278846:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177155.278848:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177155.278850:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177155.278852:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177155.278854:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177155.278855:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177155.278858:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426618:426618:256:4294967295] 192.168.202.132@tcp LPNI seq info [426580:426580:8:4294967295] 00000400:00000200:1.0:1786177155.278860:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177155.278865:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177155.278867:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177155.278869:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177155.278872:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177155.278874:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1786177155.278877:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177155.278879:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1786177155.278881:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177155.278883:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177155.278885:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177155.278888:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177155.278890:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177155.278891:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177155.278893:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177155.278895:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278901:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177155.278903:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177180 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278907:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.278908:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177155.278910:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177155.278912:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177155.278914:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177155.278915:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306502151 00000100:00000001:1.0:1786177155.278917:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306505984 00000100:00000001:1.0:1786177155.278918:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306502151 00000100:00000001:1.0:1786177155.278920:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306505984 00000100:00000001:1.0:1786177155.278921:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177155.278923:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.278924:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177155.278926:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177155/real 1786177155] req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177155.278931:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177155.278931:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177155.278933:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177155.278934:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177155.278935:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177155.278937:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177155.278939:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177155.278945:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278949:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177155.278950:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278951:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177155.278952:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177155.278954:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278958:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177155.278960:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278964:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177155.278966:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278967:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177155.278968:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278969:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177155.278970:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177155.278971:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177155.278973:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177155.278975:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177155.278977:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177155.278983:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582592:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177155.278985:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.278986:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177155.278988:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734380 x1872930235582592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177180 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177155.278993:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177155.278994:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177155.278995:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177155.278997:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177155.278998:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177155.278999:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177155.279000:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177155.279001:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:1.0:1786177155.279003:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177155.279003:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177155.279004:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734380. 00000100:00000001:1.0:1786177155.279006:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177155.279007:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.279008:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177155.279010:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177155.279011:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177155.279013:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177155.279014:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177155.789114:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177155.789124:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177155.789127:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177155.789148:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177155.789157:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177155.789162:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177155.789165:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177155.789168:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177155.789171:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177156.172501:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177156.172506:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177156.172511:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177156.172513:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177156.172517:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177156.172518:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177156.172520:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177156.172524:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177156.172526:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177156.172526:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177156.172533:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177156.172538:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177156.172542:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177156.172547:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177156.172551:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177156.172584:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177156.172589:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177156.172593:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177156.812780:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177156.812795:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177156.812799:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177156.813056:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177156.813061:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177156.813065:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177156.813068:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177156.813071:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177156.813074:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177157.196493:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177157.196501:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177157.196505:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177157.196507:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177157.196509:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177157.196517:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177157.196519:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177157.196522:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177157.196523:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177157.196527:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177157.196529:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177157.196529:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177157.196536:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177157.196540:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177157.196544:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177157.196590:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177157.196594:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177157.196597:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177157.837132:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177157.837158:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177157.837162:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177157.837170:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177157.837176:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177157.837183:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177157.837188:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177157.837191:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177157.837195:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177158.220478:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177158.220480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177158.220487:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177158.220489:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177158.220491:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177158.220492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177158.220497:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177158.220500:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177158.220502:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177158.220506:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177158.220513:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177158.220522:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177158.220525:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177158.220528:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177158.220532:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177158.220594:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177158.220601:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177158.220607:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786177158.796441:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786177158.796447:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786177158.796450:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1786177158.796655:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1786177158.796664:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1786177158.796674:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786177158.796680:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786177158.796683:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177158.860485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177158.860494:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177158.860496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177158.860503:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177158.860507:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177158.860511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177158.860514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177158.860516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177158.860519:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177159.180495:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306505984, at 4306506048 00000100:00000040:1.0:1786177159.180495:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306505984, at 4306506048 00000100:00000010:1.0:1786177159.180518:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62840. 00000100:00000010:0.0:1786177159.180520:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62540. 00000100:00000001:1.0:1786177159.244430:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177159.244438:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177159.244446:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177159.244449:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177159.244452:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177159.244453:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177159.244459:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177159.244460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177159.244463:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177159.244464:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177159.244469:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177159.244470:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177159.244472:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177159.244472:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177159.244478:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177159.244532:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177159.244536:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177159.244539:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177159.884581:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177159.884589:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177159.884591:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177159.884597:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177159.884600:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177159.884604:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177159.884606:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177159.884609:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177159.884611:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.268492:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177160.268499:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177160.268500:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177160.268503:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177160.268505:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177160.268511:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177160.268515:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177160.268519:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.268522:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177160.268525:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177160.268526:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177160.268528:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177160.268532:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177160.268536:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177160.268541:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177160.268545:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177160.268548:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177160.268551:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177160.268554:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177160.268559:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177160.268561:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177160.268622:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177160.268625:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177160.268628:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177160.396487:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177160.396501:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177160.396505:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177160.396510:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177160.396515:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177160.396525:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11834 00000400:00000200:3.0:1786177160.396536:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177160.396545:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177160.396548:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177160.396552:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.396556:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177160.396559:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.396562:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177160.396574:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177160.396578:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177160.396581:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177160.396585:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177160.396588:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177160.396591:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177160.396595:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.396597:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177160.396600:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.396602:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177160.396606:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177160.396610:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177160.396613:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177160.396616:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177160.396620:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177160.396623:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.396625:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177160.396628:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177160.396639:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786177160.396642:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177160.396647:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177160.396649:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.396653:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177160.396656:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177160.396661:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177160.396670:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786177160.396678:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177160.396714:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177160.396724:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177160.396728:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177160.396732:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177160.396756:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177160.396758:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177160.396761:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177160.396762:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177160.396763:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177160.396765:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177160.396767:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11834 00000400:00000200:3.0:1786177160.396769:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177160.396772:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177160.396773:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177160.396775:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.396777:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177160.396778:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.396780:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177160.396782:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177160.396783:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177160.396784:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177160.396786:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177160.396787:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177160.396789:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177160.396792:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.396794:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177160.396796:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.396798:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177160.396800:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177160.396801:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177160.396803:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.396804:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1786177160.396806:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177160.396812:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1786177160.396814:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177160.396817:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1786177160.396818:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.396822:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177160.396827:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177160.396830:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.396832:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177160.396835:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786177160.396836:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177160.396838:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177160.396839:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177160.396840:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177160.396841:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177160.396844:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177160.396846:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177160.396851:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786177160.396856:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.396859:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177160.396863:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177160.396864:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177160.396869:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177160.396874:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1786177160.396879:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1786177160.396881:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.396885:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177160.396894:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177160.396896:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235582720:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177160.396907:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1786177160.396913:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177160.396914:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1786177160.396918:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1786177160.396920:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177160.396926:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177160.396932:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177160.396937:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177160.396939:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177160.396943:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177160.396945:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1786177160.396946:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177160.396952:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:1.0:1786177160.396953:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000100:00080000:3.0:1786177160.396959:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11834 00000400:00000200:3.0:1786177160.396966:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177160.396968:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000400:00000200:3.0:1786177160.396971:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177160.396977:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000200:1.0:1786177160.396981:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235582720, portal 25 00000100:00000001:1.0:1786177160.396985:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177160.396991:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809984 : -110371428741632 : ffff9b9e26324e00) 00000100:00000040:3.0:1786177160.396991:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.396998:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177160.397005:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177160.397006:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177160.397011:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177160.397018:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177160.397022:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000100:00000001:1.0:1786177160.397026:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1786177160.397026:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177160.397030:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177160.397033:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177160.397037:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177160.397043:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177160.397047:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177160.397050:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000001:3.0:1786177160.397051:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1786177160.397058:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235582720, offset 0 00000100:00000040:3.0:1786177160.397064:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.397071:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177160.397072:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177160.397082:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:1.0:1786177160.397087:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:3.0:1786177160.397090:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:1.0:1786177160.397097:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:3.0:1786177160.397098:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177160.397103:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1786177160.397105:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1786177160.397108:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1786177160.397111:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 02000000:00000001:3.0:1786177160.397114:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177160.397118:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1786177160.397119:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1786177160.397123:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000008:00100000:3.0:1786177160.397125:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177160.397131:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426619:426619:256:4294967295] 192.168.202.132@tcp LPNI seq info [426581:426581:8:4294967295] 00000008:00000001:3.0:1786177160.397131:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.397135:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177160.397142:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000010:3.0:1786177160.397142:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786177160.397148:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177160.397151:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177160.397153:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.397159:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177160.397162:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1786177160.397163:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 02000000:00000001:3.0:1786177160.397167:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177160.397173:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 02000000:00000010:3.0:1786177160.397175:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000800:00000010:1.0:1786177160.397181:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000100:00000001:3.0:1786177160.397182:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177160.397192:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1786177160.397193:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177160.397198:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786177160.397209:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177160.397213:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1786177160.397214:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1786177160.397220:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1786177160.397226:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177160.397231:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1786177160.397233:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177160.397242:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1786177160.397245:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177160.397252:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177160.397259:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000100:00080000:3.0:1786177160.397262:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1786177160.397264:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00080000:3.0:1786177160.397271:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177160.397273:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786177160.397275:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177160.397279:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1786177160.397281:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177160.397284:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177160.397292:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11834 00000400:00000200:3.0:1786177160.397301:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177160.397303:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1786177160.397308:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000040:1.0:1786177160.397312:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177160.397313:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177160.397322:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177160.397328:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.397331:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1786177160.397336:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177160.397340:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177160.397343:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.397343:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177160.397350:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177160.397352:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000400:00000200:1.0:1786177160.397354:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177160.397382:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306511105 00000100:00000010:1.0:1786177160.397388:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62840. 00000020:00000001:3.0:1786177160.397395:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177160.397396:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000020:00000040:3.0:1786177160.397401:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177160.397406:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177160.397409:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306511105 00000020:00000001:3.0:1786177160.397410:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000010:1.0:1786177160.397418:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62c60. 00000100:00000040:3.0:1786177160.397419:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.397423:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177160.397428:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177160.397432:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1786177160.397433:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177160.397440:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:1.0:1786177160.397441:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177160.397447:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177160.397448:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177160.397453:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1786177160.397455:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177160.397459:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1786177160.397460:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177160/real 1786177160] req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177160.397464:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177160.397468:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.397470:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177160.397474:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177160.397481:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.397486:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177160.397493:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786177160.397499:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177160.397504:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1786177160.397506:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1786177160.397507:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1786177160.397511:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1786177160.397512:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177160.397517:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1786177160.397518:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 02000000:00000001:3.0:1786177160.397520:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177160.397522:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177160.397526:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 02000000:00000010:3.0:1786177160.397528:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:1.0:1786177160.397529:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177160.397536:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177160.397536:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177160.397549:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1786177160.397550:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177160.397570:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177160.397573:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.397582:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:3.0:1786177160.397583:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.397588:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.397595:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177160.397599:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177160.397600:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177160.397606:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1786177160.397612:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177160.397612:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177160.397616:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177160.397621:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177160.397627:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177160.397633:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11834 00000100:00000001:1.0:1786177160.397634:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1786177160.397640:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177160.397645:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1786177160.397647:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177160.397654:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177160.397661:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.397665:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177160.397668:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1786177160.397669:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.397673:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177160.397677:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177160.397678:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1786177160.397679:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177160.397682:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177160.397684:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:3.0:1786177160.397685:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:1.0:1786177160.397702:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1786177160.397704:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177160.397708:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177160.397709:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:3.0:1786177160.397712:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177160.397715:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177160.397716:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177160.397719:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177160.397724:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00080000:1.0:1786177160.397725:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:3.0:1786177160.397726:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177160.397731:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177160.397734:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177160.397737:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 02000000:00000001:3.0:1786177160.397738:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177160.397742:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177160.397744:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1786177160.397745:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.397751:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177160.397753:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000040:1.0:1786177160.397756:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1786177160.397756:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177160.397759:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177160.397766:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786177160.397771:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177160.397774:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177160.397777:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177160.397778:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235582720:192.168.202.132@tcp:250:kworker.0 00000100:00000001:3.0:1786177160.397779:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177160.397782:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177160.397785:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.397789:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1786177160.397790:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177160.397796:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.397797:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1786177160.397803:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1786177160.397807:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324e00 x1872930235582720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177160.397816:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177160.397821:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177160.397828:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.397830:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177160.397834:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1786177160.397836:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177160.397839:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 00000100:00000040:3.0:1786177160.397841:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11844) 02000000:00000001:1.0:1786177160.397846:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177160.397850:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177160.397852:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177160.397855:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177160.397859:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8400. 02000000:00000001:1.0:1786177160.397864:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177160.397866:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177160.397868:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324e00. 00000100:00000001:1.0:1786177160.397872:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177160.397874:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.397876:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.397885:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177160.397887:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.397890:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177160.397892:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177160.397893:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177160.397898:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.397907:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177160.397908:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177160.397911:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177160.397913:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177160.397915:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177160.397919:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582848:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177160.397923:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177160.397925:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177160.397927:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177160.397929:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177160.397932:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177160.397935:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177160.397940:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177160.397944:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177160.397948:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177160.397952:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235582848, portal 10 00000100:00000001:1.0:1786177160.397954:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177160.397955:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280817152 : -110371428734464 : ffff9b9e26326a00) 00000100:00000040:1.0:1786177160.397963:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.397971:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177160.397974:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177160.397976:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177160.397979:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235582848, offset 0 00000400:00000200:1.0:1786177160.397984:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177160.397988:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177160.397993:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177160.397996:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177160.397999:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177160.398002:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177160.398004:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177160.398007:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426620:426620:256:4294967295] 192.168.202.132@tcp LPNI seq info [426582:426582:8:4294967295] 00000400:00000200:1.0:1786177160.398011:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177160.398018:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177160.398022:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177160.398026:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177160.398030:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177160.398033:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786177160.398038:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177160.398041:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1786177160.398045:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177160.398047:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177160.398051:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177160.398054:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177160.398057:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177160.398059:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177160.398062:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177160.398065:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398073:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177160.398077:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398083:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.398085:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177160.398088:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177160.398093:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177160.398097:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177160.398099:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306507268 00000100:00000001:1.0:1786177160.398102:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306511105 00000100:00000001:1.0:1786177160.398104:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306507268 00000100:00000001:1.0:1786177160.398107:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306511105 00000100:00000001:1.0:1786177160.398109:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177160.398111:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.398113:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177160.398118:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177160/real 1786177160] req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177160.398127:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177160.398128:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177160.398131:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177160.398134:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177160.398136:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177160.398139:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177160.398143:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177160.398152:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398160:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177160.398161:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398163:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177160.398165:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177160.398169:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398176:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177160.398181:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398186:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177160.398188:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398189:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177160.398190:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398191:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177160.398193:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177160.398195:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177160.398197:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177160.398198:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177160.398200:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177160.398206:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582848:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177160.398209:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177160.398212:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398219:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177160.398220:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177160.398223:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177160.398225:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177160.398227:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177160.398230:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582976:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177160.398234:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177160.398236:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177160.398237:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177160.398239:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177160.398244:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177160.398246:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177160.398250:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177160.398253:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177160.398257:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177160.398260:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235582976, portal 10 00000100:00000001:1.0:1786177160.398262:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177160.398264:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280808192 : -110371428743424 : ffff9b9e26324700) 00000100:00000040:1.0:1786177160.398271:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398276:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177160.398278:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177160.398280:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177160.398282:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235582976, offset 0 00000400:00000200:1.0:1786177160.398285:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177160.398288:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177160.398291:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177160.398293:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177160.398295:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177160.398297:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177160.398299:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177160.398301:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426621:426621:256:4294967295] 192.168.202.132@tcp LPNI seq info [426583:426583:8:4294967295] 00000400:00000200:1.0:1786177160.398304:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177160.398308:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177160.398311:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177160.398313:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177160.398316:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177160.398318:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786177160.398321:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177160.398323:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1786177160.398325:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177160.398327:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177160.398329:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177160.398332:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177160.398334:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177160.398336:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177160.398338:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177160.398340:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398346:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177160.398348:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398353:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.398354:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177160.398372:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177160.398375:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177160.398377:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177160.398379:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306507269 00000100:00000001:1.0:1786177160.398381:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306511105 00000100:00000001:1.0:1786177160.398382:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306507269 00000100:00000001:1.0:1786177160.398384:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306511105 00000100:00000001:1.0:1786177160.398386:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177160.398387:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.398389:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177160.398392:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177160/real 1786177160] req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177160.398397:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177160.398399:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177160.398400:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177160.398403:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177160.398405:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177160.398407:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177160.398411:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177160.398420:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398427:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177160.398429:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398431:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177160.398432:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177160.398436:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398444:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177160.398448:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398456:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177160.398459:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398460:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177160.398462:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398464:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177160.398466:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177160.398469:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177160.398473:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177160.398475:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177160.398480:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177160.398488:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235582976:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177160.398493:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177160.398497:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398505:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177160.398506:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177160.398509:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177160.398511:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177160.398513:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177160.398517:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583104:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177160.398520:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177160.398522:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177160.398523:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177160.398524:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177160.398527:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177160.398529:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177160.398532:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177160.398534:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177160.398537:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177160.398539:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235583104, portal 4 00000100:00000001:1.0:1786177160.398541:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177160.398542:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809088 : -110371428742528 : ffff9b9e26324a80) 00000100:00000040:1.0:1786177160.398546:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398551:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177160.398553:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177160.398554:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177160.398556:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235583104, offset 0 00000400:00000200:1.0:1786177160.398559:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177160.398562:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177160.398564:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177160.398566:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177160.398568:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177160.398570:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177160.398571:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177160.398574:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426622:426622:256:4294967295] 192.168.202.132@tcp LPNI seq info [426584:426584:8:4294967295] 00000400:00000200:1.0:1786177160.398576:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177160.398581:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177160.398584:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177160.398587:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177160.398589:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177160.398591:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786177160.398594:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177160.398596:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1786177160.398598:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177160.398600:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177160.398602:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177160.398605:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177160.398607:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177160.398608:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177160.398610:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177160.398612:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398617:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177160.398619:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398628:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.398629:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177160.398632:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177160.398634:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177160.398636:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177160.398637:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306507269 00000100:00000001:1.0:1786177160.398639:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306511105 00000100:00000001:1.0:1786177160.398641:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306507269 00000100:00000001:1.0:1786177160.398643:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306511105 00000100:00000001:1.0:1786177160.398644:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177160.398646:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.398647:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177160.398651:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177160/real 1786177160] req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177160.398658:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177160.398660:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177160.398661:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177160.398664:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177160.398666:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177160.398668:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177160.398671:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177160.398679:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398699:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177160.398702:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398704:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177160.398705:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177160.398710:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398718:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177160.398723:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398729:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177160.398730:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398732:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177160.398732:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398734:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177160.398734:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177160.398736:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177160.398738:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177160.398740:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177160.398743:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177160.398748:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583104:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177160.398750:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177160.398752:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398756:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177160.398757:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177160.398759:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177160.398760:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177160.398762:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177160.398764:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583232:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177160.398766:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177160.398767:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177160.398769:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177160.398770:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177160.398773:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177160.398774:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177160.398777:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177160.398778:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177160.398781:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177160.398783:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235583232, portal 4 00000100:00000001:1.0:1786177160.398784:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177160.398785:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280806400 : -110371428745216 : ffff9b9e26324000) 00000100:00000040:1.0:1786177160.398791:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398799:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177160.398801:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177160.398804:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177160.398806:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235583232, offset 0 00000400:00000200:1.0:1786177160.398810:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177160.398814:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177160.398818:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177160.398821:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177160.398824:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177160.398826:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177160.398829:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177160.398832:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426623:426623:256:4294967295] 192.168.202.132@tcp LPNI seq info [426585:426585:8:4294967295] 00000400:00000200:1.0:1786177160.398837:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177160.398842:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177160.398845:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177160.398848:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177160.398851:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177160.398853:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1786177160.398856:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177160.398858:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1786177160.398860:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177160.398862:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177160.398865:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177160.398867:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177160.398869:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177160.398870:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177160.398872:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177160.398875:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398881:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177160.398883:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177185 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398887:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.398888:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177160.398890:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177160.398892:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177160.398895:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177160.398896:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306507271 00000100:00000001:1.0:1786177160.398898:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306511105 00000100:00000001:1.0:1786177160.398899:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306507271 00000100:00000001:1.0:1786177160.398901:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306511105 00000100:00000001:1.0:1786177160.398902:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177160.398904:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.398905:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177160.398907:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177160/real 1786177160] req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177160.398912:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177160.398912:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177160.398914:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177160.398915:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177160.398917:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177160.398918:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177160.398920:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177160.398926:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398930:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177160.398931:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398932:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177160.398933:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177160.398935:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398939:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177160.398941:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398945:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177160.398946:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398947:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177160.398948:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398949:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177160.398950:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177160.398952:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177160.398953:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177160.398955:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177160.398957:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177160.398963:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583232:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177160.398966:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398967:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177160.398970:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326a00 x1872930235582848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.398974:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177160.398975:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177160.398977:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177160.398978:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177160.398980:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177160.398981:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177160.398984:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177160.398986:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8c00. 02000000:00000001:1.0:1786177160.398988:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177160.398990:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177160.398991:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326a00. 00000100:00000001:1.0:1786177160.398994:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177160.398995:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.398997:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177160.399000:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324700 x1872930235582976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.399008:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177160.399009:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177160.399011:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177160.399013:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177160.399015:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177160.399016:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177160.399018:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177160.399019:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab800. 02000000:00000001:1.0:1786177160.399022:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177160.399024:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177160.399025:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324700. 00000100:00000001:1.0:1786177160.399027:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177160.399029:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.399031:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177160.399034:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324a80 x1872930235583104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.399041:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177160.399042:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177160.399044:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177160.399046:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177160.399047:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177160.399049:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177160.399051:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177160.399052:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa400. 02000000:00000001:1.0:1786177160.399055:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177160.399056:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177160.399057:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324a80. 00000100:00000001:1.0:1786177160.399060:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177160.399061:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.399062:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177160.399066:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324000 x1872930235583232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177185 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177160.399073:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177160.399075:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177160.399076:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177160.399078:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177160.399079:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177160.399081:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177160.399083:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177160.399085:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:1.0:1786177160.399087:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177160.399088:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177160.399090:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324000. 00000100:00000001:1.0:1786177160.399092:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177160.399094:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.399096:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.399100:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177160.399102:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.399104:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177160.399106:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177160.908481:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177160.908493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177160.908495:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177160.908503:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177160.908509:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177160.908514:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.908517:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177160.908520:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177160.908524:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177161.292453:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177161.292461:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177161.292463:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177161.292468:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177161.292469:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177161.292478:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177161.292479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177161.292483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177161.292485:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177161.292488:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177161.292491:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177161.292496:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177161.292500:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177161.292502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177161.292504:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177161.292574:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177161.292580:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177161.292584:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177161.932495:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177161.932505:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177161.932508:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177161.932516:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177161.932522:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177161.932526:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177161.932530:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177161.932534:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177161.932544:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177162.316504:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177162.316508:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177162.316513:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177162.316517:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177162.316520:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177162.316522:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177162.316523:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177162.316530:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177162.316532:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177162.316532:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177162.316539:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177162.316544:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177162.316547:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177162.316552:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177162.316556:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177162.316606:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177162.316613:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177162.316617:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177162.956498:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177162.956509:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177162.956513:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177162.956521:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177162.956527:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177162.956532:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177162.956537:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177162.956540:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177162.956543:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177163.340486:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177163.340497:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177163.340501:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177163.340501:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177163.340509:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:1.0:1786177163.340512:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177163.340515:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177163.340518:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177163.340520:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177163.340525:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177163.340525:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177163.340529:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177163.340532:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177163.340532:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177163.340536:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177163.340538:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177163.340541:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177163.340543:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177163.340545:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177163.340549:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177163.340551:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177163.340552:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177163.340554:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177163.340555:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177163.340556:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177163.340557:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177163.340559:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177163.340623:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177163.340629:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177163.340632:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177163.980496:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177163.980505:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177163.980508:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177163.980516:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177163.980522:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177163.980528:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177163.980531:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177163.980537:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177163.980542:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177164.300476:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306511105, at 4306511168 00000100:00000040:1.0:1786177164.300476:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306511105, at 4306511168 00000100:00000010:1.0:1786177164.300495:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62c60. 00000100:00000010:0.0:1786177164.300497:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62840. 00000100:00000001:1.0:1786177164.364496:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177164.364506:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177164.364506:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177164.364508:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177164.364511:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177164.364515:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177164.364517:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177164.364520:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177164.364522:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177164.364524:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177164.364526:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177164.364526:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177164.364530:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177164.364532:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177164.364535:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177164.364600:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177164.364606:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177164.364611:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177165.004496:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177165.004507:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177165.004511:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177165.004519:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177165.004524:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177165.004529:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.004532:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177165.004536:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.004539:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177165.388476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177165.388484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177165.388490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177165.388495:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177165.388496:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177165.388501:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177165.388505:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.388505:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177165.388510:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177165.388519:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177165.388525:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177165.388530:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.388535:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177165.388538:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.388543:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177165.388557:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177165.388560:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177165.388563:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177165.516513:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177165.516529:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177165.516532:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177165.516537:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177165.516543:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177165.516552:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11839 00000400:00000200:2.0:1786177165.516563:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177165.516570:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177165.516574:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177165.516577:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516580:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177165.516583:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516585:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177165.516589:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177165.516591:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177165.516594:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177165.516597:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177165.516598:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177165.516600:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177165.516602:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516603:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177165.516605:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516606:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177165.516609:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177165.516612:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516613:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177165.516616:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177165.516619:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177165.516622:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.516624:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177165.516626:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177165.516650:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 00000020:00000040:2.0:1786177165.516658:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177165.516662:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177165.516663:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.516667:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177165.516669:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177165.516674:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177165.516682:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:2.0:1786177165.516689:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177165.516698:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177165.516712:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177165.516716:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177165.516720:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177165.516741:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177165.516744:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177165.516747:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177165.516748:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177165.516750:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177165.516752:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177165.516754:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11839 00000400:00000200:2.0:1786177165.516757:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177165.516759:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177165.516761:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177165.516763:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516764:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177165.516766:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516768:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177165.516770:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177165.516771:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177165.516772:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177165.516774:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177165.516776:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177165.516777:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177165.516780:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516781:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177165.516782:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516783:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177165.516786:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177165.516788:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516791:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177165.516792:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177165.516796:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1786177165.516797:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177165.516801:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.516803:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177165.516804:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1786177165.516807:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1786177165.516808:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000008:00000001:2.0:1786177165.516811:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.516814:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177165.516816:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786177165.516820:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000100:00000040:1.0:1786177165.516822:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177165.516822:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177165.516824:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.516827:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177165.516829:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177165.516831:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177165.516837:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:2.0:1786177165.516842:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.516844:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177165.516847:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177165.516849:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177165.516854:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1786177165.516857:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1786177165.516860:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177165.516860:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177165.516866:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177165.516867:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177165.516878:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235583360:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177165.516878:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177165.516881:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177165.516888:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177165.516889:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1786177165.516890:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177165.516893:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177165.516896:0:40394:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1786177165.516898:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1786177165.516899:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11839 02000000:00000001:1.0:1786177165.516903:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177165.516903:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177165.516908:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 02000000:00000001:1.0:1786177165.516910:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1786177165.516911:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177165.516915:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516916:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:1.0:1786177165.516919:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 00000100:00000040:2.0:1786177165.516919:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516921:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:1.0:1786177165.516925:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177165.516925:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177165.516927:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177165.516929:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177165.516930:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177165.516931:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177165.516933:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177165.516935:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516936:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177165.516937:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516940:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177165.516942:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177165.516944:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177165.516947:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177165.516949:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177165.516950:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177165.516952:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177165.516955:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.516956:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:1.0:1786177165.516958:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000008:00100000:2.0:1786177165.516958:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177165.516960:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.516964:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177165.516967:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786177165.516969:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000010:1.0:1786177165.516971:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 02000000:00000001:2.0:1786177165.516971:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177165.516975:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.516977:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177165.516978:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177165.516979:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177165.516983:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235583360, portal 25 02000000:00000010:2.0:1786177165.516984:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000001:2.0:1786177165.516989:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.516990:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177165.516995:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469630848 : -110371239920768 : ffff9b9e31737b80) 00000100:00100000:2.0:1786177165.516996:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177165.517006:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177165.517009:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177165.517010:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177165.517014:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.517039:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1786177165.517040:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177165.517044:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1786177165.517049:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00080000:2.0:1786177165.517050:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177165.517053:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177165.517055:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1786177165.517056:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000001:2.0:1786177165.517058:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:1.0:1786177165.517061:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235583360, offset 0 00000100:00080000:2.0:1786177165.517062:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11839 00000400:00000200:2.0:1786177165.517067:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177165.517069:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177165.517071:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177165.517075:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000040:2.0:1786177165.517076:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.517081:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177165.517083:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.517085:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177165.517086:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177165.517087:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177165.517089:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1786177165.517091:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:2.0:1786177165.517092:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177165.517103:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000200:1.0:1786177165.517105:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000001:2.0:1786177165.517107:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177165.517112:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177165.517115:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1786177165.517116:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177165.517120:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.517122:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177165.517125:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000040:2.0:1786177165.517125:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177165.517131:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:1.0:1786177165.517135:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1786177165.517136:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177165.517140:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177165.517142:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177165.517143:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.517145:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177165.517147:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000200:1.0:1786177165.517152:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000008:00000001:2.0:1786177165.517153:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.517157:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177165.517161:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426624:426624:256:4294967295] 192.168.202.132@tcp LPNI seq info [426586:426586:8:4294967295] 00000400:00000200:1.0:1786177165.517168:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000010:2.0:1786177165.517169:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000020:00000040:2.0:1786177165.517174:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177165.517179:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177165.517183:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177165.517185:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177165.517188:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177165.517192:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177165.517194:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177165.517197:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 02000000:00000010:2.0:1786177165.517201:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177165.517205:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177165.517210:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1786177165.517212:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000100:00000001:2.0:1786177165.517220:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177165.517222:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177165.517227:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1786177165.517229:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177165.517239:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000100:00000001:2.0:1786177165.517239:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177165.517243:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1786177165.517251:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:2.0:1786177165.517255:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177165.517258:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.1:1786177165.517258:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000100:00080000:2.0:1786177165.517261:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177165.517266:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:1.0:1786177165.517270:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00080000:2.0:1786177165.517271:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11839 00000400:00000200:2.0:1786177165.517276:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177165.517278:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177165.517280:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177165.517282:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000010:1.0:1786177165.517283:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000100:00000001:2.0:1786177165.517286:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177165.517289:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.517291:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177165.517293:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1786177165.517294:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:2.0:1786177165.517295:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177165.517298:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177165.517301:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177165.517303:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177165.517306:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000200:1.0:1786177165.517309:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1786177165.517310:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177165.517314:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177165.517318:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177165.517322:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000100:00000001:2.0:1786177165.517326:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:1.0:1786177165.517329:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000020:00000040:2.0:1786177165.517329:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177165.517335:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177165.517337:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.517338:0:2392:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1786177165.517339:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177165.517343:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177165.517347:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177165.517349:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177165.517350:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177165.517354:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177165.517404:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.517407:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.517409:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000010:2.0:1786177165.517411:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734700. 00000020:00000040:2.0:1786177165.517415:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177165.517418:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1786177165.517419:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177165.517420:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.517423:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177165.517425:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177165.517428:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177165.517434:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:1.0:1786177165.517438:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177165.517440:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.517442:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1786177165.517450:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177165.517452:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177165.517461:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177165.517467:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177165.517469:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.517476:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306516225 00000100:00000040:2.0:1786177165.517477:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088313762] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1786177165.517486:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62c60. 00000100:00000001:2.0:1786177165.517490:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177165.517496:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11850) 00000100:00000001:1.0:1786177165.517497:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177165.517507:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306516225 00000100:00000010:1.0:1786177165.517511:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62ba0. 00000100:00000001:1.0:1786177165.517516:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177165.517529:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177165.517533:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.517535:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177165.517541:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177165/real 1786177165] req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177165.517553:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177165.517555:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177165.517558:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177165.517561:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177165.517563:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177165.517566:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177165.517570:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177165.517580:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.517588:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177165.517590:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.517592:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177165.517595:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177165.517599:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.517608:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177165.517613:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.517621:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177165.517624:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.517626:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177165.517627:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.517630:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177165.517632:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177165.517635:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177165.517653:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177165.517656:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177165.517662:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177165.517673:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235583360:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177165.517679:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.517681:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177165.517685:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737b80 x1872930235583360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.517693:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177165.517696:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177165.517699:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177165.517702:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177165.517704:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177165.517706:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177165.517709:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177165.517711:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:1.0:1786177165.517716:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177165.517718:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177165.517720:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737b80. 00000100:00000001:1.0:1786177165.517724:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177165.517726:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.517728:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.517737:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177165.517739:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.517742:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177165.517744:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177165.517745:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177165.517749:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.517758:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177165.517759:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177165.517762:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177165.517764:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177165.517766:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177165.517769:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583488:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177165.517773:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177165.517775:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177165.517777:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177165.517779:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177165.517782:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177165.517785:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177165.517789:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177165.517793:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177165.517796:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177165.517800:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235583488, portal 10 00000100:00000001:1.0:1786177165.517802:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177165.517803:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620992 : -110371239930624 : ffff9b9e31735500) 00000100:00000040:1.0:1786177165.517811:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.517819:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177165.517822:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177165.517824:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177165.517827:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235583488, offset 0 00000400:00000200:1.0:1786177165.517831:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177165.517836:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177165.517840:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177165.517843:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177165.517846:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177165.517849:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177165.517852:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177165.517856:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426625:426625:256:4294967295] 192.168.202.132@tcp LPNI seq info [426587:426587:8:4294967295] 00000400:00000200:1.0:1786177165.517860:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177165.517867:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177165.517872:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177165.517875:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177165.517880:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177165.517883:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786177165.517888:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177165.517891:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1786177165.517896:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177165.517898:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177165.517902:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177165.517905:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177165.517908:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177165.517910:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177165.517913:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177165.517917:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.517926:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177165.517929:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.517937:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.517939:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177165.517943:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177165.517947:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177165.517951:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177165.517953:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306512388 00000100:00000001:1.0:1786177165.517956:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306516225 00000100:00000001:1.0:1786177165.517959:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306512388 00000100:00000001:1.0:1786177165.517962:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306516225 00000100:00000001:1.0:1786177165.517964:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177165.517966:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.517968:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177165.517972:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177165/real 1786177165] req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177165.517982:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177165.517983:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177165.517985:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177165.517988:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177165.517990:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177165.517992:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177165.517996:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177165.518006:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518013:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177165.518014:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.518016:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177165.518018:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177165.518022:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518029:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177165.518033:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518042:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177165.518045:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.518046:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177165.518048:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.518050:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177165.518052:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177165.518055:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177165.518058:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177165.518060:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177165.518064:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177165.518074:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583488:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177165.518078:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177165.518082:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518089:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177165.518090:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177165.518092:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177165.518094:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177165.518097:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177165.518100:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583616:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177165.518104:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177165.518107:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177165.518108:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177165.518110:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177165.518114:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177165.518116:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177165.518120:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177165.518123:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177165.518126:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177165.518129:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235583616, portal 10 00000100:00000001:1.0:1786177165.518131:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177165.518132:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469622784 : -110371239928832 : ffff9b9e31735c00) 00000100:00000040:1.0:1786177165.518139:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518146:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177165.518149:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177165.518151:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177165.518154:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235583616, offset 0 00000400:00000200:1.0:1786177165.518158:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177165.518162:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177165.518165:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177165.518169:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177165.518171:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177165.518174:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177165.518177:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177165.518180:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426626:426626:256:4294967295] 192.168.202.132@tcp LPNI seq info [426588:426588:8:4294967295] 00000400:00000200:1.0:1786177165.518185:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177165.518192:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177165.518197:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177165.518200:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177165.518204:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177165.518208:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786177165.518213:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177165.518216:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1786177165.518220:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177165.518222:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177165.518225:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177165.518229:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177165.518232:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177165.518235:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177165.518238:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177165.518242:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518251:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177165.518255:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518262:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.518264:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177165.518267:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177165.518270:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177165.518274:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177165.518276:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306512388 00000100:00000001:1.0:1786177165.518279:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306516225 00000100:00000001:1.0:1786177165.518281:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306512388 00000100:00000001:1.0:1786177165.518284:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306516225 00000100:00000001:1.0:1786177165.518286:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177165.518288:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.518289:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177165.518293:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177165/real 1786177165] req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177165.518302:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177165.518303:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177165.518305:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177165.518308:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177165.518310:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177165.518312:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177165.518315:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177165.518325:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518333:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177165.518334:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.518336:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177165.518338:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177165.518342:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518349:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177165.518353:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518382:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177165.518385:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.518387:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177165.518388:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.518391:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177165.518393:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177165.518396:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177165.518400:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177165.518402:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177165.518407:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177165.518417:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583616:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177165.518422:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177165.518425:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518434:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177165.518435:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177165.518438:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177165.518440:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177165.518442:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177165.518446:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583744:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177165.518449:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177165.518451:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177165.518453:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177165.518455:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177165.518459:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177165.518461:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177165.518465:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177165.518468:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177165.518472:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177165.518475:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235583744, portal 4 00000100:00000001:1.0:1786177165.518477:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177165.518478:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469621888 : -110371239929728 : ffff9b9e31735880) 00000100:00000040:1.0:1786177165.518485:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518492:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177165.518494:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177165.518497:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177165.518499:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235583744, offset 0 00000400:00000200:1.0:1786177165.518503:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177165.518507:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177165.518511:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177165.518514:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177165.518517:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177165.518521:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177165.518523:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177165.518527:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426627:426627:256:4294967295] 192.168.202.132@tcp LPNI seq info [426589:426589:8:4294967295] 00000400:00000200:1.0:1786177165.518532:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177165.518539:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177165.518544:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177165.518548:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177165.518552:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177165.518556:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786177165.518560:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177165.518564:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1786177165.518568:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177165.518570:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177165.518573:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177165.518577:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177165.518580:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177165.518582:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177165.518585:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177165.518589:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518597:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177165.518601:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518608:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.518610:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177165.518613:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177165.518616:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177165.518619:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177165.518621:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306512389 00000100:00000001:1.0:1786177165.518624:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306516225 00000100:00000001:1.0:1786177165.518627:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306512389 00000100:00000001:1.0:1786177165.518629:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306516225 00000100:00000001:1.0:1786177165.518631:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177165.518634:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.518635:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177165.518653:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177165/real 1786177165] req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177165.518663:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177165.518664:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177165.518667:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177165.518669:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177165.518671:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177165.518674:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177165.518678:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177165.518688:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518695:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177165.518697:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.518699:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177165.518701:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177165.518705:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518713:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177165.518717:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518724:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177165.518727:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.518728:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177165.518730:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.518731:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177165.518733:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177165.518736:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177165.518739:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177165.518741:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177165.518746:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177165.518755:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583744:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177165.518759:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177165.518762:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518770:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177165.518771:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177165.518773:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177165.518775:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177165.518777:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177165.518780:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583872:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177165.518784:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177165.518786:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177165.518787:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177165.518790:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177165.518793:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177165.518795:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177165.518799:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177165.518802:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000400:00000010:1.0:1786177165.518805:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177165.518808:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235583872, portal 4 00000100:00000001:1.0:1786177165.518810:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177165.518812:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469617408 : -110371239934208 : ffff9b9e31734700) 00000100:00000040:1.0:1786177165.518819:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518826:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177165.518828:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177165.518831:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00000200:1.0:1786177165.518833:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235583872, offset 0 00000400:00000200:1.0:1786177165.518838:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177165.518843:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177165.518847:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177165.518850:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177165.518853:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177165.518856:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177165.518858:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177165.518862:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426628:426628:256:4294967295] 192.168.202.132@tcp LPNI seq info [426590:426590:8:4294967295] 00000400:00000200:1.0:1786177165.518867:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177165.518873:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177165.518878:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177165.518881:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000800:00000200:1.0:1786177165.518886:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177165.518889:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1786177165.518893:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177165.518896:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1786177165.518900:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177165.518902:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:1.0:1786177165.518905:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177165.518909:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177165.518913:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:1.0:1786177165.518914:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:1.0:1786177165.518917:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177165.518921:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518930:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177165.518933:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177190 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.518941:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.518942:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177165.518945:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177165.518948:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177165.518951:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177165.518954:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306512391 00000100:00000001:1.0:1786177165.518957:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306516225 00000100:00000001:1.0:1786177165.518959:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306512391 00000100:00000001:1.0:1786177165.518962:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306516225 00000100:00000001:1.0:1786177165.518964:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177165.518967:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.518969:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177165.518974:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177165/real 1786177165] req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177165.518983:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177165.518984:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177165.518986:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177165.518989:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177165.518991:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177165.518993:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177165.518997:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177165.519006:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.519013:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177165.519015:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.519016:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177165.519018:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177165.519022:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.519029:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177165.519033:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.519040:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177165.519043:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.519045:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177165.519047:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.519048:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177165.519050:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177165.519053:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177165.519056:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177165.519058:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177165.519062:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177165.519070:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235583872:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177165.519074:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.519076:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177165.519079:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735500 x1872930235583488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.519087:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177165.519089:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177165.519091:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177165.519093:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177165.519095:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177165.519097:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177165.519099:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177165.519102:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:1.0:1786177165.519105:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177165.519107:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177165.519108:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735500. 00000100:00000001:1.0:1786177165.519111:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177165.519112:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.519114:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177165.519118:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735c00 x1872930235583616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.519132:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177165.519134:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177165.519136:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177165.519138:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177165.519139:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177165.519141:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177165.519143:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177165.519144:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:1.0:1786177165.519147:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177165.519148:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177165.519149:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735c00. 00000100:00000001:1.0:1786177165.519151:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177165.519152:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.519154:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177165.519157:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735880 x1872930235583744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.519164:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177165.519166:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177165.519167:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177165.519170:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177165.519171:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177165.519172:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177165.519174:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177165.519176:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:1.0:1786177165.519178:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177165.519179:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177165.519180:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735880. 00000100:00000001:1.0:1786177165.519183:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177165.519184:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.519186:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177165.519189:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734700 x1872930235583872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177190 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177165.519196:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177165.519197:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177165.519199:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177165.519201:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177165.519203:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177165.519204:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177165.519207:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177165.519209:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:1.0:1786177165.519211:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177165.519212:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177165.519214:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734700. 00000100:00000001:1.0:1786177165.519216:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177165.519218:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.519220:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177165.519224:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177165.519226:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177165.519228:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177165.519230:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177166.028416:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177166.028424:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177166.028427:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177166.028432:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177166.028435:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177166.028439:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177166.028441:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177166.028443:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177166.028445:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177166.412447:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177166.412452:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177166.412455:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177166.412455:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177166.412458:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177166.412460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177166.412462:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177166.412462:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177166.412467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177166.412467:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177166.412468:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177166.412471:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177166.412473:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177166.412476:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177166.412479:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177166.412529:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177166.412533:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177166.412535:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177167.052453:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177167.052460:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177167.052462:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177167.052467:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177167.052472:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177167.052475:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177167.052478:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177167.052480:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177167.052482:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177167.436455:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177167.436458:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177167.436462:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177167.436463:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177167.436465:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177167.436470:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177167.436471:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177167.436473:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177167.436477:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177167.436478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177167.436481:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177167.436482:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177167.436485:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177167.436487:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177167.436489:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177167.436540:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177167.436544:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177167.436546:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177168.076477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177168.076482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177168.076484:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177168.076488:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177168.076491:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177168.076494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177168.076496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177168.076498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177168.076500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177168.460465:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177168.460469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177168.460473:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177168.460474:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177168.460478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177168.460480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177168.460483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177168.460485:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177168.460486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177168.460488:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177168.460493:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177168.460498:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177168.460502:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177168.460505:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177168.460508:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177168.460545:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177168.460550:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177168.460554:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177169.100461:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177169.100467:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177169.100469:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177169.100473:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177169.100475:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177169.100478:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177169.100480:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177169.100483:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177169.100485:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177169.420463:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306516225, at 4306516288 00000100:00000040:1.0:1786177169.420463:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306516225, at 4306516288 00000100:00000010:0.0:1786177169.420477:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62c60. 00000100:00000010:1.0:1786177169.420480:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ba0. 00000100:00000001:1.0:1786177169.484459:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177169.484462:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177169.484466:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177169.484467:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177169.484470:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177169.484473:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177169.484475:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177169.484476:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177169.484481:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177169.484481:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177169.484486:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177169.484487:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177169.484489:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177169.484493:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177169.484494:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177169.484495:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177169.484500:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177169.484504:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177169.484507:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177169.484512:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177169.484514:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177169.484574:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177169.484577:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177169.484580:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177170.124460:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177170.124469:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177170.124472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177170.124478:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177170.124484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177170.124488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177170.124492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177170.124494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177170.124498:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.508451:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177170.508453:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177170.508457:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.508460:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177170.508463:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177170.508464:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177170.508466:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177170.508469:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177170.508471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177170.508474:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.508474:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177170.508479:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177170.508482:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177170.508485:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177170.508487:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177170.508532:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177170.508536:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177170.508538:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177170.636518:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177170.636526:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177170.636528:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177170.636531:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177170.636536:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177170.636542:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11845 00000400:00000200:2.0:1786177170.636550:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177170.636555:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177170.636558:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177170.636561:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636563:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177170.636566:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636567:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177170.636571:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177170.636572:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177170.636574:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177170.636577:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177170.636579:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177170.636580:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177170.636582:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636584:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177170.636585:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636586:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177170.636603:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177170.636605:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636607:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177170.636609:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177170.636612:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177170.636615:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.636616:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177170.636618:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177170.636627:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737480. 00000020:00000040:2.0:1786177170.636629:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177170.636633:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177170.636634:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.636637:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177170.636639:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177170.636643:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177170.636649:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9000. 00000100:00000001:2.0:1786177170.636654:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177170.636661:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177170.636669:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177170.636673:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177170.636676:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046bba14a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177170.636704:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177170.636707:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177170.636709:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177170.636711:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177170.636712:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177170.636714:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177170.636716:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11845 00000400:00000200:2.0:1786177170.636719:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177170.636721:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177170.636723:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177170.636725:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636727:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177170.636729:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636731:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177170.636733:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177170.636734:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177170.636735:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177170.636737:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177170.636738:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177170.636739:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177170.636742:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1786177170.636743:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636747:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177170.636748:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1786177170.636749:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177170.636751:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1786177170.636752:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177170.636755:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177170.636759:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000040:1.0:1786177170.636761:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177170.636761:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177170.636764:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177170.636770:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177170.636774:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.636778:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177170.636783:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1786177170.636784:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177170.636787:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1786177170.636788:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.636791:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177170.636792:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177170.636795:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1786177170.636795:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736d80. 00000020:00000040:2.0:1786177170.636799:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177170.636800:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177170.636803:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177170.636806:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177170.636807:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235584000:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177170.636809:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177170.636812:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177170.636813:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177170.636814:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177170.636818:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:2.0:1786177170.636818:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aabc00. 02000000:00000001:1.0:1786177170.636821:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.636823:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177170.636824:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1786177170.636829:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1786177170.636830:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177170.636835:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177170.636843:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177170.636843:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177170.636848:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177170.636849:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068428f0. 00000100:00000040:2.0:1786177170.636858:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e4f7686] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1786177170.636859:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177170.636866:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235584000, portal 25 00000100:00000001:1.0:1786177170.636870:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177170.636873:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469629056 : -110371239922560 : ffff9b9e31737480) 00000100:00000001:2.0:1786177170.636875:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177170.636881:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177170.636881:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177170.636892:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177170.636896:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177170.636897:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1786177170.636900:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1786177170.636904:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636906:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:1.0:1786177170.636909:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e660. 00000100:00080000:2.0:1786177170.636912:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11845 00000100:00000200:1.0:1786177170.636915:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235584000, offset 0 00000400:00000200:2.0:1786177170.636920:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177170.636927:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177170.636928:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177170.636934:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177170.636938:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636941:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177170.636942:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177170.636946:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636950:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177170.636954:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000001:2.0:1786177170.636955:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177170.636958:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000400:00000200:1.0:1786177170.636962:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1786177170.636962:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177170.636966:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:1.0:1786177170.636969:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000001:2.0:1786177170.636970:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177170.636974:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1786177170.636975:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177170.636980:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000040:2.0:1786177170.636981:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.636985:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177170.636989:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177170.636990:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426629:426629:256:4294967295] 192.168.202.132@tcp LPNI seq info [426591:426591:8:4294967295] 00000100:00000001:2.0:1786177170.636991:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177170.636996:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:1.0:1786177170.637000:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:2.0:1786177170.637001:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177170.637004:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177170.637006:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177170.637009:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177170.637013:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177170.637015:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177170.637016:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177170.637019:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177170.637022:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.637024:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177170.637027:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000010:2.0:1786177170.637030:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734700. 00000020:00000040:2.0:1786177170.637033:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177170.637035:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177170.637035:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.637037:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177170.637038:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177170.637039:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177170.637042:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:2.0:1786177170.637045:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177170.637046:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e100. 00000100:00100000:2.0:1786177170.637051:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1786177170.637060:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177170.637063:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177170.637068:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786177170.637069:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:2.0:1786177170.637076:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088313762] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177170.637085:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.637087:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177170.637091:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177170.637092:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177170.637099:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306521344 00000100:00080000:2.0:1786177170.637099:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177170.637102:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177170.637105:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:1.0:1786177170.637108:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62ba0. 00000100:00000001:2.0:1786177170.637109:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177170.637115:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11845 00000100:00000001:1.0:1786177170.637118:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000400:00000200:2.0:1786177170.637121:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177170.637124:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177170.637126:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177170.637129:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177170.637130:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306521344 00000100:00000001:2.0:1786177170.637131:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177170.637135:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.637138:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000010:1.0:1786177170.637139:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62900. 00000020:00000001:2.0:1786177170.637141:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177170.637144:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000020:00000040:2.0:1786177170.637144:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177170.637147:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177170.637150:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177170.637151:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177170.637153:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177170.637156:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.637157:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177170.637159:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1786177170.637160:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177170.637163:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.637164:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177170.637169:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:1.0:1786177170.637170:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177170.637171:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177170.637174:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177170.637175:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177170.637180:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177170.637183:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177170.637185:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1786177170.637185:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.637190:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1786177170.637190:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177170.637192:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177170.637196:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:2.0:1786177170.637196:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.637197:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.637200:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:2.0:1786177170.637203:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000100:00000001:1.0:1786177170.637204:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1786177170.637206:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177170.637209:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177170.637211:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177170.637213:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177170.637214:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177170.637216:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177170.637219:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177170.637225:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000001:2.0:1786177170.637229:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.637231:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177170.637234:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177170.637235:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177170.637238:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177170.637242:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177170.637248:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.637254:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1786177170.637258:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584128:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177170.637258:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177170.637264:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177170.637275:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1786177170.637278:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177170.637280:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1786177170.637281:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1786177170.637283:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177170.637285:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177170.637289:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1786177170.637290:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1786177170.637291:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177170.637294:0:40394:0:(import.c:485:import_select_connection()) Process entered 02000000:00000010:1.0:1786177170.637295:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae400. 00000100:00080000:2.0:1786177170.637298:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11845 02000000:00000001:1.0:1786177170.637299:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177170.637304:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177170.637307:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177170.637309:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:2.0:1786177170.637310:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177170.637314:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177170.637316:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842d68. 00000100:00000001:2.0:1786177170.637317:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177170.637319:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.637322:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177170.637324:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177170.637325:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000400:00000010:1.0:1786177170.637326:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e198. 00000020:00000001:2.0:1786177170.637327:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177170.637329:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177170.637332:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000200:1.0:1786177170.637333:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235584128, portal 10 00000020:00000001:2.0:1786177170.637333:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177170.637339:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1786177170.637339:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.637341:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177170.637343:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469627264 : -110371239924352 : ffff9b9e31736d80) 00000100:00000040:2.0:1786177170.637344:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177170.637347:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177170.637350:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177170.637353:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177170.637355:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177170.637381:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1786177170.637384:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177170.637384:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177170.637388:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.637391:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177170.637393:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177170.637397:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.637401:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.637406:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:2.0:1786177170.637406:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786177170.637410:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000040:1.0:1786177170.637415:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 02000000:00000001:2.0:1786177170.637415:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177170.637418:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177170.637420:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177170.637421:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1786177170.637422:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ed48. 02000000:00000001:2.0:1786177170.637423:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177170.637428:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000200:1.0:1786177170.637429:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235584128, offset 0 00000100:00000001:2.0:1786177170.637431:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177170.637439:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177170.637442:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177170.637449:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177170.637456:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:2.0:1786177170.637456:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177170.637461:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177170.637462:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177170.637469:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:2.0:1786177170.637472:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177170.637483:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1786177170.637483:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177170.637490:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000040:2.0:1786177170.637492:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11855) 00000400:00000200:1.0:1786177170.637497:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426630:426630:255:4294967295] 192.168.202.132@tcp LPNI seq info [426592:426592:7:4294967295] 00000400:00000200:1.0:1786177170.637503:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177170.637513:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177170.637518:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177170.637523:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e200. 00000800:00000200:1.0:1786177170.637527:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177170.637531:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177170.637534:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177170.637538:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177170.637542:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177170.637544:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306517508 00000100:00000001:1.0:1786177170.637547:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306521344 00000100:00000001:1.0:1786177170.637549:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306517508 00000100:00000001:1.0:1786177170.637552:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306521344 00000100:00000001:1.0:1786177170.637553:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177170.637555:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.637558:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177170.637567:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177170.637575:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177170.637576:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177170.637578:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177170.637580:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177170.637582:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177170.637585:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584256:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177170.637604:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177170.637607:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177170.637608:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177170.637610:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177170.637614:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177170.637616:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177170.637619:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177170.637622:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842f78. 00000400:00000010:1.0:1786177170.637626:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177170.637628:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235584256, portal 10 00000100:00000001:1.0:1786177170.637630:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177170.637631:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469617408 : -110371239934208 : ffff9b9e31734700) 00000100:00000040:1.0:1786177170.637652:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177170.637660:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177170.637662:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177170.637665:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e2a8. 00000100:00000200:1.0:1786177170.637668:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235584256, offset 0 00000400:00000200:1.0:1786177170.637672:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177170.637676:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177170.637680:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177170.637683:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177170.637686:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177170.637689:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177170.637691:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177170.637694:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426631:426631:254:4294967295] 192.168.202.132@tcp LPNI seq info [426593:426593:6:4294967295] 00000400:00000200:1.0:1786177170.637698:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177170.637704:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177170.637708:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177170.637711:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142ec00. 00000800:00000200:1.0:1786177170.637716:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177170.637720:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177170.637723:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177170.637726:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177170.637730:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177170.637732:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306517508 00000100:00000001:1.0:1786177170.637734:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306521344 00000100:00000001:1.0:1786177170.637736:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306517508 00000100:00000001:1.0:1786177170.637738:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306521344 00000100:00000001:1.0:1786177170.637741:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177170.637743:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.637745:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.637747:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177170.637751:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177170.637754:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1786177170.637756:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177170.637757:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177170.637759:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177170.637763:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177170.637771:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177170.637772:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177170.637775:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177170.637777:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177170.637779:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177170.637783:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584384:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177170.637787:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177170.637788:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177170.637790:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177170.637792:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177170.637795:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177170.637798:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177170.637801:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177170.637804:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177170.637808:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177170.637811:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235584384, portal 4 00000100:00000001:1.0:1786177170.637813:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177170.637814:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469621888 : -110371239929728 : ffff9b9e31735880) 00000100:00000040:1.0:1786177170.637821:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177170.637829:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177170.637832:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177170.637834:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e3b8. 00000100:00000200:1.0:1786177170.637837:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235584384, offset 0 00000400:00000200:1.0:1786177170.637841:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177170.637845:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177170.637848:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177170.637852:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177170.637854:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177170.637857:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177170.637860:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177170.637863:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426632:426632:253:4294967295] 192.168.202.132@tcp LPNI seq info [426594:426594:5:4294967295] 00000400:00000200:1.0:1786177170.637867:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177170.637873:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177170.637879:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177170.637882:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e000. 00000800:00000200:1.0:1786177170.637887:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177170.637891:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177170.637894:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177170.637897:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177170.637900:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177170.637902:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306517508 00000100:00000001:1.0:1786177170.637904:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306521344 00000100:00000001:1.0:1786177170.637906:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306517508 00000100:00000001:1.0:1786177170.637908:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306521344 00000100:00000001:1.0:1786177170.637910:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177170.637912:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.637914:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177170.637918:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177170.637925:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177170.637926:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177170.637929:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177170.637931:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177170.637933:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177170.637936:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584512:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177170.637939:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177170.637941:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177170.637943:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177170.637944:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177170.637947:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177170.637949:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177170.637952:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177170.637955:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177170.637958:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177170.637961:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235584512, portal 4 00000100:00000001:1.0:1786177170.637963:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177170.637964:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469622784 : -110371239928832 : ffff9b9e31735c00) 00000100:00000040:1.0:1786177170.637970:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177170.637978:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177170.637980:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177170.637983:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e7f8. 00000100:00000200:1.0:1786177170.637986:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235584512, offset 0 00000400:00000200:1.0:1786177170.637990:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177170.637994:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177170.637998:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177170.638000:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177170.638003:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177170.638005:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177170.638008:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177170.638010:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426633:426633:252:4294967295] 192.168.202.132@tcp LPNI seq info [426595:426595:4:4294967295] 00000400:00000200:1.0:1786177170.638015:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177170.638021:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177170.638026:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177170.638029:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177170.638034:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177170.638037:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177170.638040:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177170.638043:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177170.638045:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177170.638047:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306517510 00000100:00000001:1.0:1786177170.638050:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306521344 00000100:00000001:1.0:1786177170.638052:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306517510 00000100:00000001:1.0:1786177170.638055:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306521344 00000100:00000001:1.0:1786177170.638057:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177170.638059:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.638062:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.638064:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177170.638067:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177170.638069:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1786177170.638071:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177170.638072:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177170.638074:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177170.638076:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177171.148461:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177171.148477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177171.148480:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177171.148486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177171.148492:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177171.148496:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177171.148500:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177171.148504:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177171.148508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177171.532461:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177171.532468:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177171.532471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177171.532471:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177171.532477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177171.532478:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177171.532482:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177171.532485:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177171.532486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177171.532487:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177171.532490:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177171.532491:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177171.532497:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177171.532501:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177171.532503:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177171.532550:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177171.532553:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177171.532556:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177172.172458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177172.172465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177172.172467:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177172.172472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177172.172476:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177172.172480:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177172.172483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177172.172486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177172.172490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177172.556464:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177172.556466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177172.556472:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177172.556472:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177172.556476:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177172.556477:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177172.556480:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177172.556484:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177172.556486:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177172.556492:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177172.556492:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177172.556497:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177172.556498:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177172.556500:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177172.556503:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177172.556508:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177172.556509:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177172.556512:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177172.556514:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177172.556518:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177172.556522:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177172.556524:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177172.556526:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177172.556528:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177172.556529:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177172.556532:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177172.556534:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177172.556618:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177172.556623:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177172.556627:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177173.196443:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177173.196451:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177173.196454:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177173.196459:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177173.196464:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177173.196468:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.196471:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177173.196474:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.196477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177173.580458:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177173.580462:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177173.580465:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177173.580466:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177173.580468:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177173.580471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177173.580471:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177173.580472:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177173.580475:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177173.580477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177173.580478:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177173.580487:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.580490:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177173.580492:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.580495:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177173.580526:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177173.580529:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177173.580532:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1786177173.708653:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:0.0:1786177173.708697:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1786177173.708714:0:2383:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177173.708719:0:2383:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:0.0:1786177173.708724:0:2383:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177173.708728:0:2383:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1786177173.708734:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177173.708738:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e100. 00000400:00000200:0.0:1786177173.708744:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177173.708748:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177173.708755:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e660 00000400:00000010:0.0:1786177173.708758:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e660. 00000100:00000001:0.0:1786177173.708763:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177173.708772:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177173.708801:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177173.708806:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177173.708815:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177173.708817:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786177173.708829:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177173.708832:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e200. 00000400:00000200:0.0:1786177173.708836:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177173.708840:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177173.708844:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ed48 00000400:00000010:0.0:1786177173.708846:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ed48. 00000100:00000001:0.0:1786177173.708850:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177173.708855:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.708862:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177173.708864:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1786177173.708869:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1786177173.708870:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.708875:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1786177173.708882:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177173.708886:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1786177173.708888:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177170/real 1786177173] req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1786177173.708896:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177173.708903:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142ec00. 00000400:00000200:0.0:1786177173.708908:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000001:1.0:1786177173.708913:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:0.0:1786177173.708914:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000001:1.0:1786177173.708919:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177173.708923:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842d68. 00000400:00000200:1.0:1786177173.708928:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e198 00000400:00000010:1.0:1786177173.708930:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e198. 00000100:00000001:1.0:1786177173.708933:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:0.0:1786177173.708934:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e2a8 00000400:00000010:0.0:1786177173.708940:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e2a8. 00000100:00000200:1.0:1786177173.708941:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177173.708947:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177173.708956:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177173.708959:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177173.708983:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177173.708990:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.708990:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177173.708995:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709001:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177173.709004:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177173.709010:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177173.709011:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177173.709017:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177173.709024:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000100:0.0:1786177173.709029:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177173.709035:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e000. 00000100:00000040:1.0:1786177173.709035:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1786177173.709039:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177173.709043:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177173.709046:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e3b8 00000400:00000010:0.0:1786177173.709051:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e3b8. 00000100:00000001:0.0:1786177173.709057:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1786177173.709062:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000200:0.0:1786177173.709066:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709068:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709072:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177173.709075:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709080:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177173.709087:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1786177173.709087:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1786177173.709094:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177173.709095:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177173.709105:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177173.709108:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177173.709110:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1786177173.709111:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1786177173.709121:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1786177173.709122:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177173.709130:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:0.0:1786177173.709135:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:1.0:1786177173.709141:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584128:192.168.202.132@tcp:38:kworker.0 00000400:00000200:0.0:1786177173.709144:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177173.709153:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e7f8 00000400:00000010:0.0:1786177173.709155:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e7f8. 00000100:00000001:0.0:1786177173.709157:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177173.709162:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709167:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1786177173.709173:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1786177173.709177:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177170/real 1786177173] req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786177173.709182:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177195 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177173.709190:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177173.709192:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177173.709194:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842f78. 00000400:00000200:1.0:1786177173.709198:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177173.709199:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177173.709202:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177173.709207:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177173.709215:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709223:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177173.709224:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709226:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177173.709228:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177173.709232:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177173.709263:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177173.709264:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177173.709267:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1786177173.709273:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709282:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177173.709285:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709288:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177173.709290:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709292:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177173.709294:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177173.709298:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177173.709302:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177173.709305:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177173.709310:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177173.709319:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584256:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177173.709324:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177173.709328:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177170/real 1786177173] req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177173.709336:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177173.709338:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177173.709340:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068428f0. 00000400:00000200:1.0:1786177173.709342:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177173.709344:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177173.709347:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177173.709350:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177173.709386:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709394:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177173.709396:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709398:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177173.709399:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177173.709403:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709416:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177173.709421:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709428:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177173.709431:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709433:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177173.709434:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709436:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177173.709438:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177173.709441:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177173.709445:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177173.709448:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177173.709452:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177173.709460:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235584000:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177173.709464:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177173.709466:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177173.709470:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736d80 x1872930235584128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709482:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177173.709485:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177173.709489:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177173.709494:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177173.709500:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177173.709503:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177173.709505:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177173.709508:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:1.0:1786177173.709513:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177173.709515:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177173.709517:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736d80. 00000100:00000001:1.0:1786177173.709521:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177173.709522:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709524:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177173.709528:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734700 x1872930235584256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709534:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177173.709535:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177173.709537:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177173.709539:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177173.709540:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177173.709541:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177173.709543:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177173.709545:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:1.0:1786177173.709547:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177173.709548:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177173.709549:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734700. 00000100:00000001:1.0:1786177173.709551:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177173.709552:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709554:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177173.709557:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737480 x1872930235584000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709575:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177173.709576:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177173.709578:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177173.709581:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177173.709582:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177173.709583:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177173.709584:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177173.709585:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:1.0:1786177173.709588:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177173.709589:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177173.709590:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737480. 00000100:00000001:1.0:1786177173.709593:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177173.709594:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709597:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177173.709600:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177173.709601:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177173.709603:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177173.709607:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177170/real 1786177173] req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177173.709615:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177173.709617:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177173.709621:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177173.709624:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177173.709626:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177173.709629:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177173.709632:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177173.709642:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709650:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177173.709652:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709654:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177173.709656:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177173.709661:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709668:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177173.709672:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709680:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177173.709682:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709684:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177173.709685:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709687:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177173.709689:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177173.709693:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177173.709697:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177173.709700:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177173.709704:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177173.709713:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584384:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177173.709717:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177173.709721:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177170/real 1786177173] req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177173.709738:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177173.709739:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177173.709741:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177173.709744:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177173.709746:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177173.709748:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177173.709752:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177173.709761:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709767:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177173.709768:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709770:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177173.709771:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177173.709775:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709781:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177173.709786:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709798:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177173.709801:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709803:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177173.709804:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709806:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177173.709808:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177173.709810:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177173.709814:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177173.709816:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177173.709820:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177173.709829:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584512:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177173.709833:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709835:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177173.709838:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735880 x1872930235584384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709844:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177173.709846:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177173.709847:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177173.709850:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177173.709852:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177173.709853:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177173.709855:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177173.709857:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:1.0:1786177173.709859:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177173.709860:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177173.709862:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735880. 00000100:00000001:1.0:1786177173.709864:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177173.709866:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709868:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177173.709871:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735c00 x1872930235584512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177195 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177173.709877:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177173.709878:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177173.709880:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177173.709882:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177173.709883:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177173.709884:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177173.709886:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177173.709887:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:1.0:1786177173.709889:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177173.709890:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177173.709891:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735c00. 00000100:00000001:1.0:1786177173.709893:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177173.709895:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709896:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177173.709904:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177173.709906:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177173.709908:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177173.709910:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177174.220467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177174.220474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177174.220476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177174.220482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177174.220486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177174.220490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177174.220492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177174.220494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177174.220496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177174.540479:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306521344, at 4306521408 00000100:00000040:1.0:1786177174.540479:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306521344, at 4306521408 00000100:00000010:1.0:1786177174.540494:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62900. 00000100:00000010:0.0:1786177174.540496:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ba0. 00000100:00000001:1.0:1786177174.604459:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177174.604462:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177174.604466:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177174.604469:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177174.604471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177174.604473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177174.604474:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177174.604480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177174.604482:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177174.604483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177174.604488:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177174.604493:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177174.604497:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177174.604500:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177174.604503:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177174.604541:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177174.604546:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177174.604563:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177175.244472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177175.244479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177175.244482:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177175.244488:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177175.244491:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177175.244495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.244498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177175.244501:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.244504:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.628455:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177175.628461:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177175.628463:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177175.628467:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177175.628470:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177175.628471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177175.628473:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177175.628476:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.628478:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177175.628483:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177175.628484:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177175.628486:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177175.628487:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177175.628492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177175.628493:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177175.628581:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177175.628586:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177175.628590:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177175.756481:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177175.756489:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177175.756492:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177175.756494:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177175.756498:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177175.756504:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11850 00000400:00000200:3.0:1786177175.756511:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177175.756516:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177175.756519:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177175.756522:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177175.756524:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177175.756526:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177175.756528:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177175.756532:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177175.756533:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177175.756535:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177175.756553:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177175.756556:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177175.756557:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177175.756560:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177175.756561:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177175.756562:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177175.756564:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177175.756566:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177175.756568:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177175.756570:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177175.756572:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177175.756576:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177175.756579:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.756582:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177175.756584:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177175.756592:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786177175.756596:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177175.756600:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177175.756602:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.756606:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177175.756608:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177175.756613:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177175.756621:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786177175.756627:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177175.756638:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177175.756648:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177175.756654:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177175.756659:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177175.756688:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177175.756691:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177175.756695:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177175.756697:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177175.756698:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177175.756701:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177175.756703:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11850 00000400:00000200:3.0:1786177175.756707:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177175.756711:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177175.756714:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177175.756717:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177175.756720:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177175.756723:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177175.756725:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177175.756728:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177175.756729:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177175.756731:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177175.756732:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1786177175.756734:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00000001:1.0:1786177175.756738:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1786177175.756739:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177175.756743:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1786177175.756744:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177175.756753:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177175.756756:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177175.756756:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177175.756762:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177175.756767:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177175.756773:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:1.0:1786177175.756778:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1786177175.756780:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177175.756782:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.756784:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177175.756788:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177175.756789:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177175.756794:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177175.756795:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177175.756798:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177175.756800:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.756804:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:1.0:1786177175.756808:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235584640:192.168.202.132@tcp:250:kworker.0 00000008:00100000:3.0:1786177175.756811:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1786177175.756816:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:3.0:1786177175.756817:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.756821:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177175.756822:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:3.0:1786177175.756826:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 02000000:00000001:1.0:1786177175.756827:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177175.756832:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177175.756839:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177175.756842:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1786177175.756848:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177175.756850:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:3.0:1786177175.756855:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1786177175.756858:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 02000000:00000001:3.0:1786177175.756859:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.756865:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177175.756867:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177175.756869:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177175.756870:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 02000000:00000010:3.0:1786177175.756876:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000200:1.0:1786177175.756881:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235584640, portal 25 00000100:00000001:3.0:1786177175.756882:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.756888:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177175.756893:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280811776 : -110371428739840 : ffff9b9e26325500) 00000100:00100000:3.0:1786177175.756894:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1786177175.756905:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177175.756908:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177175.756914:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.756922:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1786177175.756924:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177175.756931:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177175.756936:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000001:3.0:1786177175.756941:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177175.756942:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235584640, offset 0 00000100:00080000:3.0:1786177175.756948:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177175.756953:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1786177175.756955:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177175.756957:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177175.756974:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177175.756979:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177175.756984:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11850 00000400:00000200:1.0:1786177175.756988:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177175.756990:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177175.756998:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177175.757002:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00080000:3.0:1786177175.757003:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177175.757011:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177175.757013:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1786177175.757016:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177175.757022:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177175.757024:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:3.0:1786177175.757026:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177175.757031:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1786177175.757032:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177175.757035:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000400:00000200:1.0:1786177175.757040:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000001:3.0:1786177175.757040:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177175.757045:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:1.0:1786177175.757049:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426634:426634:256:4294967295] 192.168.202.132@tcp LPNI seq info [426596:426596:8:4294967295] 00000020:00000001:3.0:1786177175.757050:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177175.757057:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177175.757062:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177175.757066:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177175.757069:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177175.757073:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177175.757076:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177175.757080:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177175.757088:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:1.0:1786177175.757092:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:3.0:1786177175.757094:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177175.757100:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177175.757103:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1786177175.757107:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 02000000:00000001:3.0:1786177175.757108:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.757113:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177175.757115:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000800:00000010:1.0:1786177175.757117:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000008:00000001:3.0:1786177175.757118:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.757123:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177175.757129:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000800:00000200:1.0:1786177175.757131:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786177175.757133:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000800:00000200:1.0:1786177175.757139:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 02000000:00000001:3.0:1786177175.757139:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177175.757142:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.757147:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177175.757149:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1786177175.757152:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786177175.757153:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786177175.757160:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 02000000:00000010:3.0:1786177175.757162:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786177175.757169:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1786177175.757170:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177175.757176:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00100000:3.0:1786177175.757179:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177175.757184:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177175.757194:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1786177175.757194:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177175.757200:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177175.757204:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000100:00000040:3.0:1786177175.757207:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1786177175.757209:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177175.757216:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786177175.757221:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177175.757225:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177175.757227:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177175.757236:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177175.757240:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177175.757248:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1786177175.757251:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1786177175.757254:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177175.757258:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177175.757266:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11850 00000100:00000001:1.0:1786177175.757268:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.757272:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1786177175.757275:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177175.757281:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1786177175.757281:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177175.757287:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177175.757290:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177175.757294:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177175.757296:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306526464 00000100:00000010:1.0:1786177175.757305:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62900. 00000100:00000001:1.0:1786177175.757312:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177175.757319:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306526464 00000100:00000010:1.0:1786177175.757322:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62060. 00000100:00000001:1.0:1786177175.757325:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177175.757329:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177175.757332:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.757334:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177175.757339:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177175/real 1786177175] req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177175.757349:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177175.757351:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177175.757353:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177175.757388:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177175.757390:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177175.757393:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177175.757398:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177175.757409:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.757417:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177175.757420:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177175.757422:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177175.757425:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177175.757430:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177175.757430:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177175.757436:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00100000:1.0:1786177175.757441:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177175.757442:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177175.757445:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177175.757449:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177175.757453:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:1.0:1786177175.757455:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1786177175.757457:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177175.757462:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:1.0:1786177175.757464:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177175.757469:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177175.757475:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177175.757480:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177175.757484:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177175.757485:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177175.757491:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1786177175.757492:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:1.0:1786177175.757496:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177175.757500:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177175.757500:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177175.757506:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1786177175.757507:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.757511:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1786177175.757512:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177175.757517:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1786177175.757518:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1786177175.757524:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177175.757528:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:3.0:1786177175.757529:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177175.757534:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177175.757535:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:3.0:1786177175.757552:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.757556:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177175.757559:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1786177175.757562:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786177175.757567:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177175.757573:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1786177175.757577:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235584640:192.168.202.132@tcp:250:kworker.0 02000000:00000001:3.0:1786177175.757578:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.757582:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177175.757586:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177175.757588:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1786177175.757589:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.757594:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:3.0:1786177175.757598:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000040:1.0:1786177175.757601:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325500 x1872930235584640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177175.757604:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177175.757615:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177175.757635:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1786177175.757637:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1786177175.757640:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1786177175.757642:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177175.757644:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac000. 00000100:00000040:3.0:1786177175.757649:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177175.757651:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177175.757655:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177175.757658:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177175.757664:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177175.757669:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786177175.757670:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177175.757675:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:3.0:1786177175.757676:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1786177175.757680:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:3.0:1786177175.757682:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000010:1.0:1786177175.757684:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325500. 00000100:00000001:3.0:1786177175.757686:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177175.757690:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177175.757691:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1786177175.757694:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177175.757695:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.757700:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177175.757701:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11850 00000400:00000200:3.0:1786177175.757708:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177175.757710:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1786177175.757713:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177175.757715:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177175.757718:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177175.757720:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177175.757724:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1786177175.757725:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177175.757727:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177175.757729:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177175.757736:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177175.757738:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177175.757739:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177175.757746:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177175.757749:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177175.757752:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177175.757756:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:1.0:1786177175.757757:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177175.757761:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177175.757761:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 02000000:00000001:1.0:1786177175.757766:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1786177175.757768:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 02000000:00000001:1.0:1786177175.757771:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1786177175.757775:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177175.757777:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.757781:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1786177175.757786:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584768:192.168.202.132@tcp:38:kworker.0 00000100:00000040:3.0:1786177175.757789:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177175.757794:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1786177175.757795:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:1.0:1786177175.757799:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1786177175.757800:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 02000000:00000001:1.0:1786177175.757804:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177175.757807:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:1.0:1786177175.757808:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1786177175.757813:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177175.757815:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac000. 02000000:00000001:3.0:1786177175.757816:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177175.757821:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177175.757822:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177175.757827:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177175.757831:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177175.757833:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177175.757837:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000010:1.0:1786177175.757838:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000008:00000001:3.0:1786177175.757844:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177175.757847:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000001:3.0:1786177175.757848:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177175.757854:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235584768, portal 10 00000100:00000010:3.0:1786177175.757856:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000100:00000001:1.0:1786177175.757859:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000020:00000040:3.0:1786177175.757861:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177175.757863:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280812672 : -110371428738944 : ffff9b9e26325880) 02000000:00000001:3.0:1786177175.757866:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177175.757870:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177175.757873:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:1.0:1786177175.757875:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177175.757876:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177175.757879:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177175.757887:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:1.0:1786177175.757889:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1786177175.757893:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177175.757895:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177175.757900:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00100000:3.0:1786177175.757902:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1786177175.757905:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235584768, offset 0 00000400:00000200:1.0:1786177175.757917:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177175.757918:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177175.757922:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177175.757925:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177175.757931:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177175.757935:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177175.757942:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177175.757948:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:3.0:1786177175.757950:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177175.757953:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177175.757959:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000040:3.0:1786177175.757959:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11860) 00000400:00000200:1.0:1786177175.757965:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426635:426635:256:4294967295] 192.168.202.132@tcp LPNI seq info [426597:426597:8:4294967295] 00000400:00000200:1.0:1786177175.757971:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177175.757980:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177175.757986:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177175.757991:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177175.757996:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177175.757999:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177175.758004:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177175.758007:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177175.758011:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177175.758014:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177175.758017:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177175.758021:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177175.758025:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177175.758027:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177175.758031:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177175.758035:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758045:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177175.758048:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758056:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.758058:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177175.758062:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177175.758066:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177175.758070:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177175.758073:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306522628 00000100:00000001:1.0:1786177175.758075:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306526464 00000100:00000001:1.0:1786177175.758077:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306522628 00000100:00000001:1.0:1786177175.758078:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306526464 00000100:00000001:1.0:1786177175.758080:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177175.758081:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.758083:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177175.758086:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177175/real 1786177175] req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177175.758091:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177175.758092:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177175.758094:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177175.758096:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177175.758098:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177175.758100:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177175.758104:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177175.758112:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758125:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177175.758127:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758129:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177175.758131:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177175.758135:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758143:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177175.758147:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758154:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177175.758157:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758159:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177175.758161:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758163:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177175.758165:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177175.758168:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177175.758172:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177175.758175:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177175.758180:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177175.758189:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584768:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177175.758193:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177175.758197:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758205:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177175.758206:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177175.758209:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177175.758211:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177175.758214:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177175.758216:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584896:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177175.758220:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177175.758222:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177175.758224:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177175.758226:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177175.758229:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177175.758232:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177175.758235:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177175.758238:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177175.758242:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177175.758245:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235584896, portal 10 00000100:00000001:1.0:1786177175.758247:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177175.758248:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280821632 : -110371428729984 : ffff9b9e26327b80) 00000100:00000040:1.0:1786177175.758254:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758260:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177175.758263:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177175.758265:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177175.758268:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235584896, offset 0 00000400:00000200:1.0:1786177175.758272:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177175.758276:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177175.758280:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177175.758283:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177175.758286:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177175.758288:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177175.758291:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177175.758294:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426636:426636:256:4294967295] 192.168.202.132@tcp LPNI seq info [426598:426598:8:4294967295] 00000400:00000200:1.0:1786177175.758298:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177175.758304:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177175.758310:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177175.758314:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177175.758318:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177175.758321:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177175.758325:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177175.758328:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177175.758332:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177175.758334:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177175.758337:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177175.758341:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177175.758344:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177175.758346:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177175.758349:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177175.758353:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758384:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177175.758387:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758391:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.758392:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177175.758396:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177175.758398:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177175.758401:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177175.758402:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306522629 00000100:00000001:1.0:1786177175.758404:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306526464 00000100:00000001:1.0:1786177175.758406:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306522629 00000100:00000001:1.0:1786177175.758407:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306526464 00000100:00000001:1.0:1786177175.758408:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177175.758410:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.758411:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177175.758414:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177175/real 1786177175] req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177175.758420:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177175.758421:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177175.758422:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177175.758424:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177175.758425:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177175.758426:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177175.758428:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177175.758437:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758444:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177175.758446:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758447:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177175.758449:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177175.758452:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758459:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177175.758464:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758471:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177175.758474:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758476:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177175.758477:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758479:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177175.758482:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177175.758485:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177175.758489:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177175.758491:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177175.758496:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177175.758511:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235584896:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177175.758516:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177175.758520:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758527:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177175.758529:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177175.758533:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177175.758535:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177175.758537:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177175.758553:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585024:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177175.758557:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177175.758559:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177175.758561:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177175.758563:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177175.758567:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177175.758569:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177175.758573:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177175.758576:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177175.758578:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177175.758581:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235585024, portal 4 00000100:00000001:1.0:1786177175.758583:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177175.758584:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280815360 : -110371428736256 : ffff9b9e26326300) 00000100:00000040:1.0:1786177175.758591:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758598:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177175.758600:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177175.758602:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177175.758605:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235585024, offset 0 00000400:00000200:1.0:1786177175.758610:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177175.758615:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177175.758618:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177175.758622:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177175.758624:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177175.758627:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177175.758631:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177175.758634:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426637:426637:256:4294967295] 192.168.202.132@tcp LPNI seq info [426599:426599:8:4294967295] 00000400:00000200:1.0:1786177175.758639:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177175.758646:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177175.758651:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177175.758655:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177175.758659:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177175.758662:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177175.758667:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177175.758670:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177175.758674:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177175.758676:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177175.758679:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177175.758682:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177175.758686:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177175.758687:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177175.758690:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177175.758694:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758702:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177175.758706:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758713:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.758715:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177175.758719:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177175.758723:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177175.758726:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177175.758729:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306522629 00000100:00000001:1.0:1786177175.758731:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306526464 00000100:00000001:1.0:1786177175.758734:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306522629 00000100:00000001:1.0:1786177175.758736:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306526464 00000100:00000001:1.0:1786177175.758739:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177175.758742:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.758744:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177175.758749:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177175/real 1786177175] req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177175.758758:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177175.758760:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177175.758761:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177175.758764:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177175.758766:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177175.758769:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177175.758773:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177175.758782:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758789:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177175.758791:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758794:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177175.758795:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177175.758799:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758807:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177175.758812:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758827:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177175.758831:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758832:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177175.758834:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758836:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177175.758837:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177175.758840:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177175.758844:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177175.758846:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177175.758850:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177175.758859:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585024:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177175.758864:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758866:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177175.758870:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325880 x1872930235584768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758876:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177175.758878:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177175.758881:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177175.758884:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177175.758886:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177175.758888:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177175.758890:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177175.758892:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:1.0:1786177175.758897:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177175.758899:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177175.758901:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325880. 00000100:00000001:1.0:1786177175.758904:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177175.758905:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758907:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177175.758910:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327b80 x1872930235584896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758919:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177175.758920:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177175.758922:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177175.758924:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177175.758925:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177175.758925:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177175.758927:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177175.758928:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:1.0:1786177175.758930:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177175.758931:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177175.758933:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327b80. 00000100:00000001:1.0:1786177175.758935:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177175.758937:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758939:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177175.758944:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326300 x1872930235585024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758950:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177175.758951:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177175.758953:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177175.758956:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177175.758959:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177175.758961:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177175.758963:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177175.758964:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 02000000:00000001:1.0:1786177175.758967:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177175.758968:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177175.758970:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326300. 00000100:00000001:1.0:1786177175.758972:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177175.758974:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758976:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.758982:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177175.758984:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.758987:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177175.758989:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177175.758990:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177175.758994:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.758999:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177175.759001:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177175.759003:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177175.759004:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177175.759005:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177175.759007:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585152:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177175.759010:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177175.759012:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177175.759013:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177175.759015:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177175.759018:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177175.759020:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177175.759024:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177175.759026:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177175.759030:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177175.759033:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235585152, portal 4 00000100:00000001:1.0:1786177175.759035:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177175.759036:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280813568 : -110371428738048 : ffff9b9e26325c00) 00000100:00000040:1.0:1786177175.759042:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.759049:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177175.759052:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177175.759055:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177175.759057:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235585152, offset 0 00000400:00000200:1.0:1786177175.759061:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177175.759066:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177175.759070:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177175.759072:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177175.759074:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177175.759076:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177175.759077:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177175.759080:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426638:426638:256:4294967295] 192.168.202.132@tcp LPNI seq info [426600:426600:8:4294967295] 00000400:00000200:1.0:1786177175.759083:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177175.759088:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177175.759091:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177175.759093:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177175.759096:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177175.759099:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177175.759103:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177175.759106:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177175.759110:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177175.759112:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177175.759115:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177175.759118:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177175.759121:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177175.759123:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177175.759125:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177175.759129:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.759138:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177175.759141:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177200 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.759149:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.759151:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177175.759154:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177175.759158:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177175.759161:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177175.759163:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306522631 00000100:00000001:1.0:1786177175.759166:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306526464 00000100:00000001:1.0:1786177175.759168:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306522631 00000100:00000001:1.0:1786177175.759169:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306526464 00000100:00000001:1.0:1786177175.759171:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177175.759173:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.759174:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177175.759177:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177175/real 1786177175] req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177175.759183:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177175.759184:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177175.759185:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177175.759187:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177175.759188:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177175.759190:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177175.759192:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177175.759197:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.759202:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177175.759203:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.759204:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177175.759205:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177175.759207:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.759211:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177175.759213:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.759218:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177175.759219:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.759220:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177175.759221:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.759223:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177175.759224:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177175.759225:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177175.759229:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177175.759231:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177175.759235:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177175.759243:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585152:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177175.759247:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.759248:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177175.759252:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325c00 x1872930235585152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177200 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177175.759259:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177175.759260:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177175.759263:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177175.759265:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177175.759266:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177175.759268:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177175.759270:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177175.759272:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:1.0:1786177175.759274:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177175.759275:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177175.759277:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325c00. 00000100:00000001:1.0:1786177175.759280:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177175.759281:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.759283:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177175.759286:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177175.759287:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177175.759288:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177175.759290:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177176.268473:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177176.268481:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177176.268483:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177176.268490:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177176.268495:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177176.268501:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177176.268505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177176.268509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177176.268512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177176.652431:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177176.652438:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177176.652439:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177176.652443:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177176.652443:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177176.652450:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177176.652451:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177176.652453:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177176.652455:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177176.652457:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177176.652459:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177176.652459:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177176.652463:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177176.652467:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177176.652469:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177176.652525:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177176.652529:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177176.652557:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177177.292484:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177177.292493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177177.292496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177177.292503:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177177.292508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177177.292512:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177177.292516:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177177.292519:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177177.292535:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177177.676462:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177177.676469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177177.676469:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177177.676472:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177177.676475:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177177.676478:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177177.676482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177177.676483:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177177.676486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177177.676488:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177177.676492:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177177.676495:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177177.676495:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177177.676500:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177177.676500:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177177.676590:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177177.676595:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177177.676599:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177178.316488:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177178.316497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177178.316501:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177178.316507:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177178.316512:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177178.316530:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177178.316534:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177178.316537:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177178.316541:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177178.700416:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177178.700422:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177178.700423:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177178.700429:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177178.700431:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177178.700434:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177178.700437:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177178.700438:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177178.700441:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177178.700443:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177178.700446:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177178.700447:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177178.700449:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177178.700450:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177178.700454:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177178.700506:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177178.700529:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177178.700532:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177179.340476:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177179.340484:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177179.340486:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177179.340493:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177179.340497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177179.340502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177179.340519:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177179.340522:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177179.340525:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177179.660462:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306526464, at 4306526528 00000100:00000040:0.0:1786177179.660463:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306526464, at 4306526528 00000100:00000010:0.0:1786177179.660478:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62900. 00000100:00000010:1.0:1786177179.660481:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62060. 00010000:00000001:0.0:1786177179.724475:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177179.724475:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177179.724480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177179.724482:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177179.724486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177179.724487:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177179.724490:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:1.0:1786177179.724493:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177179.724494:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177179.724498:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177179.724512:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177179.724517:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177179.724518:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177179.724521:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177179.724523:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177179.724525:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177179.724528:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177179.724528:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177179.724535:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177179.724536:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177179.724539:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177179.724592:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177179.724596:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177179.724599:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177180.364480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177180.364490:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177180.364492:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177180.364513:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177180.364517:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177180.364521:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.364524:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177180.364528:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.364531:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.748426:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177180.748434:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177180.748435:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177180.748439:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177180.748440:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177180.748445:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.748446:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177180.748451:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177180.748452:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177180.748454:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177180.748455:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.748457:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177180.748460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177180.748461:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177180.748462:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177180.748537:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177180.748541:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177180.748544:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786177180.876512:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177180.876521:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786177180.876523:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177180.876526:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177180.876529:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177180.876536:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11855 00000400:00000200:0.0:1786177180.876543:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177180.876548:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177180.876551:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177180.876553:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876556:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177180.876558:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876559:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177180.876563:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177180.876565:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786177180.876568:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177180.876570:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786177180.876572:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786177180.876574:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786177180.876576:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876577:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177180.876578:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876579:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177180.876582:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786177180.876584:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876585:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177180.876587:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177180.876590:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177180.876592:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.876593:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177180.876595:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177180.876604:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786177180.876606:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786177180.876609:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177180.876611:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.876614:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177180.876616:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177180.876619:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177180.876627:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786177180.876633:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177180.876640:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177180.876648:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177180.876651:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177180.876655:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177180.876678:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177180.876681:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177180.876684:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786177180.876685:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177180.876686:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177180.876688:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177180.876690:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11855 00000400:00000200:0.0:1786177180.876692:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177180.876694:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177180.876695:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177180.876698:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876700:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177180.876702:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876704:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177180.876706:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177180.876707:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786177180.876708:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177180.876710:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786177180.876711:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786177180.876713:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177180.876714:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1786177180.876715:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876718:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177180.876719:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1786177180.876720:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177180.876723:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1786177180.876724:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177180.876728:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786177180.876733:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000040:1.0:1786177180.876734:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177180.876735:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177180.876741:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177180.876746:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177180.876747:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177180.876750:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177180.876751:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.876754:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1786177180.876755:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177180.876758:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:0.0:1786177180.876759:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1786177180.876763:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1786177180.876764:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.876768:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177180.876770:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235585280:192.168.202.132@tcp:250:kworker.0 00000100:00000010:0.0:1786177180.876772:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000100:00000001:1.0:1786177180.876775:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1786177180.876776:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177180.876779:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177180.876780:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177180.876782:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177180.876783:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.876788:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177180.876788:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1786177180.876792:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177180.876793:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177180.876795:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac000. 02000000:00000010:0.0:1786177180.876797:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:1.0:1786177180.876799:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.876803:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177180.876807:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:0.0:1786177180.876810:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177180.876812:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000100:00000001:0.0:1786177180.876820:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1786177180.876820:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000001:0.0:1786177180.876823:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177180.876827:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177180.876828:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235585280, portal 25 00000100:00000001:1.0:1786177180.876834:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177180.876837:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794204288 : -110371915347328 : ffff9b9e09314a80) 00000100:00000001:0.0:1786177180.876839:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177180.876842:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1786177180.876845:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1786177180.876846:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786177180.876849:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177180.876851:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177180.876857:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177180.876861:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1786177180.876862:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11855 00000400:00000200:0.0:1786177180.876868:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177180.876870:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177180.876872:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177180.876874:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000040:0.0:1786177180.876875:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876878:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1786177180.876879:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000040:0.0:1786177180.876882:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:1.0:1786177180.876884:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235585280, offset 0 00000100:00000001:0.0:1786177180.876885:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177180.876889:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177180.876890:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786177180.876892:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177180.876895:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:1.0:1786177180.876895:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000020:00000001:0.0:1786177180.876899:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786177180.876901:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786177180.876904:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177180.876907:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177180.876908:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177180.876910:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876911:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177180.876913:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786177180.876917:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876919:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177180.876920:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1786177180.876923:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 02000000:00000001:0.0:1786177180.876924:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177180.876928:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177180.876930:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1786177180.876931:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177180.876934:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177180.876936:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000008:00000001:0.0:1786177180.876938:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.876940:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177180.876940:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000010:0.0:1786177180.876945:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000400:00000200:1.0:1786177180.876945:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000040:0.0:1786177180.876950:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177180.876952:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426639:426639:256:4294967295] 192.168.202.132@tcp LPNI seq info [426601:426601:8:4294967295] 02000000:00000001:0.0:1786177180.876953:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177180.876956:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.876958:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1786177180.876959:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:0.0:1786177180.876960:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177180.876963:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177180.876967:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000400:00000200:1.0:1786177180.876971:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:0.0:1786177180.876973:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177180.876978:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1786177180.876984:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:0.0:1786177180.876990:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177180.876993:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177180.876994:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000040:0.0:1786177180.877001:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1786177180.877005:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177180.877012:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1786177180.877013:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177180.877018:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1786177180.877023:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:0.0:1786177180.877024:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786177180.877028:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.1:1786177180.877028:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00080000:0.0:1786177180.877029:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177180.877035:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:1.0:1786177180.877038:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00080000:0.0:1786177180.877041:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11855 00000800:00000010:1.0:1786177180.877043:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:0.0:1786177180.877047:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177180.877049:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177180.877050:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177180.877053:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177180.877057:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1786177180.877061:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.877063:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177180.877064:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000100:00000040:0.0:1786177180.877066:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177180.877068:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:0.0:1786177180.877070:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177180.877073:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177180.877073:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:0.0:1786177180.877075:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786177180.877077:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:1.0:1786177180.877080:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1786177180.877081:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786177180.877085:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786177180.877089:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786177180.877095:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.877098:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177180.877099:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177180.877102:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.877107:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:1.0:1786177180.877107:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1786177180.877112:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786177180.877118:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177180.877124:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177180.877128:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177180.877131:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177180.877133:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177180.877134:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.877139:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177180.877140:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177180.877143:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786177180.877145:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.877146:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.877147:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177180.877153:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000400:00000200:1.0:1786177180.877154:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:0.0:1786177180.877156:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177180.877159:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177180.877160:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.877161:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177180.877161:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177180.877163:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.877163:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306531584 02000000:00000010:0.0:1786177180.877170:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000010:1.0:1786177180.877171:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62060. 00000100:00000001:0.0:1786177180.877173:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177180.877178:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177180.877179:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177180.877186:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177180.877188:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.877189:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306531584 00000100:00000040:0.0:1786177180.877193:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1786177180.877194:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62b40. 00000100:00000001:1.0:1786177180.877199:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177180.877204:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177180.877207:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177180.877209:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177180.877214:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177180.877214:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.877218:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177180.877218:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1786177180.877221:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177180.877224:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:1.0:1786177180.877226:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177180/real 1786177180] req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1786177180.877228:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11855 00000400:00000200:0.0:1786177180.877236:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177180.877242:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177180.877245:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000001:1.0:1786177180.877246:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:0.0:1786177180.877249:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000001:1.0:1786177180.877250:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177180.877253:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000100:00000001:0.0:1786177180.877254:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177180.877258:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000100:00000040:0.0:1786177180.877259:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.877263:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:1.0:1786177180.877263:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000020:00000001:0.0:1786177180.877268:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177180.877268:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:0.0:1786177180.877270:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786177180.877273:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:1.0:1786177180.877274:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1786177180.877275:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786177180.877280:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786177180.877283:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786177180.877290:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.877292:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1786177180.877294:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786177180.877295:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177180.877299:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177180.877303:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786177180.877307:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177180.877308:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177180.877309:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177180.877311:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177180.877315:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.877319:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177180.877323:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177180.877324:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00000001:0.0:1786177180.877325:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.877328:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.877329:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1786177180.877331:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786177180.877334:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177180.877334:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1786177180.877338:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177180.877339:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1786177180.877340:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177180.877345:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177180.877346:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177180.877349:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177180.877350:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1786177180.877353:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:1.0:1786177180.877391:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177180.877392:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177180.877396:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177180.877399:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177180.877409:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177180.877412:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177180.877419:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877419:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177180.877424:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.877428:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177180.877430:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177180.877432:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.877433:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177180.877436:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1786177180.877437:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11865) 00000100:00080000:1.0:1786177180.877439:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177180.877445:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177180.877447:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177180.877451:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177180.877457:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235585280:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177180.877461:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.877463:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177180.877465:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314a80 x1872930235585280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877470:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177180.877472:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177180.877474:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177180.877476:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177180.877477:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177180.877479:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177180.877480:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177180.877483:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:1.0:1786177180.877485:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177180.877487:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177180.877503:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314a80. 00000100:00000001:1.0:1786177180.877508:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177180.877509:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.877511:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.877516:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177180.877517:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.877519:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177180.877520:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177180.877521:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177180.877524:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877530:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177180.877531:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177180.877532:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177180.877534:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177180.877535:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177180.877537:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09315500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585408:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177180.877539:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177180.877541:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177180.877542:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177180.877543:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177180.877546:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177180.877547:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177180.877550:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177180.877552:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177180.877555:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177180.877557:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235585408, portal 10 00000100:00000001:1.0:1786177180.877559:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177180.877560:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794206976 : -110371915344640 : ffff9b9e09315500) 00000100:00000040:1.0:1786177180.877565:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877569:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177180.877571:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177180.877573:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177180.877574:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235585408, offset 0 00000400:00000200:1.0:1786177180.877577:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177180.877580:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177180.877583:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177180.877585:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177180.877587:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177180.877589:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177180.877590:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177180.877593:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426640:426640:256:4294967295] 192.168.202.132@tcp LPNI seq info [426602:426602:8:4294967295] 00000400:00000200:1.0:1786177180.877595:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177180.877600:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177180.877603:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177180.877605:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177180.877608:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177180.877610:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786177180.877613:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177180.877615:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1786177180.877619:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177180.877620:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177180.877623:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177180.877635:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177180.877639:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177180.877641:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177180.877644:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177180.877648:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877656:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177180.877660:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877667:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.877669:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177180.877673:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177180.877678:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177180.877680:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177180.877682:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306527748 00000100:00000001:1.0:1786177180.877684:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306531584 00000100:00000001:1.0:1786177180.877686:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306527748 00000100:00000001:1.0:1786177180.877687:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306531584 00000100:00000001:1.0:1786177180.877688:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177180.877690:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.877691:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177180.877694:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177180/real 1786177180] req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177180.877699:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177180.877700:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177180.877702:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177180.877703:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177180.877704:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177180.877706:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177180.877708:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177180.877714:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877718:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177180.877719:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.877720:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177180.877721:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177180.877723:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877727:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177180.877729:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877735:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177180.877737:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.877738:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177180.877739:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.877740:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177180.877742:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177180.877743:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177180.877745:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177180.877747:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177180.877749:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177180.877754:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09315500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585408:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177180.877757:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177180.877759:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877764:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177180.877765:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177180.877766:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177180.877767:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177180.877768:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177180.877770:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585536:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177180.877772:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177180.877774:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177180.877775:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177180.877776:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177180.877779:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177180.877780:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177180.877782:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177180.877784:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177180.877786:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177180.877788:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235585536, portal 10 00000100:00000001:1.0:1786177180.877790:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177180.877791:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215040 : -110371915336576 : ffff9b9e09317480) 00000100:00000040:1.0:1786177180.877795:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877800:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177180.877802:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177180.877803:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177180.877805:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235585536, offset 0 00000400:00000200:1.0:1786177180.877807:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177180.877810:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177180.877812:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177180.877814:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177180.877815:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177180.877817:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177180.877819:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177180.877821:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426641:426641:256:4294967295] 192.168.202.132@tcp LPNI seq info [426603:426603:8:4294967295] 00000400:00000200:1.0:1786177180.877825:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177180.877829:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177180.877832:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177180.877834:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177180.877837:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177180.877839:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786177180.877842:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177180.877844:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1786177180.877846:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177180.877847:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177180.877849:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177180.877852:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177180.877854:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177180.877856:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177180.877857:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177180.877859:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877865:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177180.877867:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877871:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.877872:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177180.877874:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177180.877876:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177180.877878:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177180.877879:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306527748 00000100:00000001:1.0:1786177180.877881:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306531584 00000100:00000001:1.0:1786177180.877883:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306527748 00000100:00000001:1.0:1786177180.877884:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306531584 00000100:00000001:1.0:1786177180.877885:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177180.877886:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.877887:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177180.877889:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177180/real 1786177180] req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177180.877894:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177180.877895:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177180.877896:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177180.877897:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177180.877898:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177180.877900:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177180.877902:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177180.877907:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877911:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177180.877912:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.877914:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177180.877914:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177180.877917:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877921:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177180.877923:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877927:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177180.877928:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.877929:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177180.877930:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.877931:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177180.877932:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177180.877934:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177180.877935:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177180.877937:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177180.877939:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177180.877944:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585536:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177180.877946:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177180.877948:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877952:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177180.877953:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177180.877955:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177180.877956:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177180.877957:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177180.877959:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585664:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177180.877961:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177180.877962:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177180.877963:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177180.877964:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177180.877966:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177180.877967:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177180.877969:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177180.877971:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177180.877973:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177180.877975:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235585664, portal 4 00000100:00000001:1.0:1786177180.877976:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177180.877977:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794213248 : -110371915338368 : ffff9b9e09316d80) 00000100:00000040:1.0:1786177180.877981:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.877985:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177180.877987:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177180.877988:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177180.877989:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235585664, offset 0 00000400:00000200:1.0:1786177180.877992:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177180.877993:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177180.877995:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177180.877997:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177180.877998:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177180.878000:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177180.878001:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177180.878003:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426642:426642:256:4294967295] 192.168.202.132@tcp LPNI seq info [426604:426604:8:4294967295] 00000400:00000200:1.0:1786177180.878006:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177180.878010:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177180.878013:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177180.878015:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177180.878017:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177180.878019:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786177180.878022:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177180.878023:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1786177180.878026:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177180.878027:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177180.878029:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177180.878031:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177180.878033:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177180.878034:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177180.878035:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177180.878037:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878042:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177180.878044:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878048:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.878049:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177180.878051:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177180.878052:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177180.878054:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177180.878055:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306527748 00000100:00000001:1.0:1786177180.878056:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306531584 00000100:00000001:1.0:1786177180.878058:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306527748 00000100:00000001:1.0:1786177180.878059:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306531584 00000100:00000001:1.0:1786177180.878061:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177180.878062:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.878063:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177180.878065:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177180/real 1786177180] req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177180.878070:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177180.878071:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177180.878072:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177180.878073:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177180.878074:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177180.878075:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177180.878077:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177180.878082:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878086:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177180.878087:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878088:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177180.878088:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177180.878090:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878094:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177180.878097:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878100:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177180.878102:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878103:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177180.878104:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878105:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177180.878106:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177180.878107:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177180.878109:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177180.878111:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177180.878114:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177180.878119:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585664:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177180.878121:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177180.878123:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878127:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177180.878128:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177180.878129:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177180.878130:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177180.878131:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177180.878133:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585792:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177180.878134:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177180.878135:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177180.878136:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177180.878137:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177180.878139:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177180.878140:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177180.878142:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177180.878143:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177180.878145:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177180.878147:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235585792, portal 4 00000100:00000001:1.0:1786177180.878148:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177180.878149:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794203392 : -110371915348224 : ffff9b9e09314700) 00000100:00000040:1.0:1786177180.878152:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878156:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177180.878157:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177180.878159:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177180.878161:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235585792, offset 0 00000400:00000200:1.0:1786177180.878163:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177180.878165:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177180.878167:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177180.878169:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177180.878170:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177180.878172:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177180.878173:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177180.878175:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426643:426643:256:4294967295] 192.168.202.132@tcp LPNI seq info [426605:426605:8:4294967295] 00000400:00000200:1.0:1786177180.878178:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177180.878182:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177180.878184:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177180.878186:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177180.878188:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177180.878190:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786177180.878192:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177180.878194:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1786177180.878196:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177180.878197:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177180.878199:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177180.878201:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177180.878203:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177180.878204:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177180.878206:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177180.878208:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878212:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177180.878214:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177205 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878218:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.878219:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177180.878220:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177180.878222:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177180.878224:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177180.878225:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306527750 00000100:00000001:1.0:1786177180.878227:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306531584 00000100:00000001:1.0:1786177180.878228:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306527750 00000100:00000001:1.0:1786177180.878230:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306531584 00000100:00000001:1.0:1786177180.878231:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177180.878232:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.878233:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177180.878235:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177180/real 1786177180] req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177180.878239:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177180.878240:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177180.878241:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177180.878242:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177180.878243:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177180.878245:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177180.878247:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177180.878252:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878256:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177180.878256:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878257:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177180.878258:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177180.878260:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878264:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177180.878266:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878270:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177180.878272:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878273:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177180.878274:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878275:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177180.878276:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177180.878278:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177180.878280:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177180.878281:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177180.878283:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177180.878290:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235585792:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177180.878293:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878295:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177180.878298:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09315500 x1872930235585408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878304:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177180.878306:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177180.878308:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177180.878310:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177180.878312:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177180.878313:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177180.878315:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177180.878317:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:1.0:1786177180.878320:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177180.878321:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177180.878323:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09315500. 00000100:00000001:1.0:1786177180.878326:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177180.878328:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878329:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177180.878333:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317480 x1872930235585536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878345:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177180.878346:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177180.878347:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177180.878349:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177180.878349:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177180.878350:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177180.878351:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177180.878352:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:1.0:1786177180.878354:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177180.878355:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177180.878370:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317480. 00000100:00000001:1.0:1786177180.878373:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177180.878374:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878375:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177180.878378:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316d80 x1872930235585664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878382:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177180.878383:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177180.878384:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177180.878386:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177180.878386:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177180.878387:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177180.878389:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177180.878390:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:1.0:1786177180.878391:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177180.878391:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177180.878392:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316d80. 00000100:00000001:1.0:1786177180.878394:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177180.878395:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878396:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177180.878398:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314700 x1872930235585792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177205 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177180.878402:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177180.878403:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177180.878404:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177180.878406:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177180.878407:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177180.878408:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177180.878409:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177180.878410:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:1.0:1786177180.878411:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177180.878411:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177180.878412:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314700. 00000100:00000001:1.0:1786177180.878414:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177180.878414:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878416:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177180.878418:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177180.878419:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177180.878420:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177180.878421:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177181.388482:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177181.388501:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177181.388504:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177181.388510:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177181.388515:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177181.388519:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177181.388522:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177181.388525:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177181.388529:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177181.772460:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177181.772467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177181.772468:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177181.772470:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177181.772473:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177181.772476:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177181.772479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177181.772495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177181.772497:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177181.772500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177181.772502:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177181.772504:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177181.772506:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177181.772509:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177181.772511:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177181.772574:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177181.772579:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177181.772582:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177182.412489:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177182.412499:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177182.412502:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177182.412508:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177182.412515:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177182.412520:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177182.412523:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177182.412527:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177182.412531:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177182.796487:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177182.796496:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177182.796498:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177182.796502:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177182.796507:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177182.796510:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177182.796512:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177182.796515:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177182.796519:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177182.796521:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177182.796524:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177182.796526:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177182.796528:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177182.796529:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177182.796532:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177182.796533:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177182.796535:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177182.796537:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177182.796599:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177182.796603:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177182.796606:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177182.796661:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177182.796666:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177182.796669:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177182.796673:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177182.796677:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177182.796680:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177182.796683:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177182.796686:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177182.796689:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177183.436483:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177183.436492:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177183.436495:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177183.436501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177183.436506:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177183.436510:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177183.436514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177183.436516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177183.436520:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177183.820451:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177183.820456:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177183.820473:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177183.820473:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177183.820478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177183.820481:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177183.820485:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177183.820485:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177183.820491:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177183.820492:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177183.820494:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177183.820498:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177183.820503:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177183.820505:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177183.820509:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177183.820560:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177183.820564:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177183.820573:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177184.460492:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177184.460499:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177184.460502:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177184.460507:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177184.460511:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177184.460515:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177184.460519:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177184.460522:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177184.460525:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177184.780492:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306531584, at 4306531648 00000100:00000040:0.0:1786177184.780495:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306531584, at 4306531648 00000100:00000010:1.0:1786177184.780510:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62b40. 00000100:00000010:0.0:1786177184.780512:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62060. 00000100:00000001:1.0:1786177184.844463:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177184.844467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177184.844470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177184.844471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177184.844473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177184.844477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177184.844477:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177184.844480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177184.844482:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177184.844484:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177184.844486:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177184.844487:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177184.844491:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177184.844494:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177184.844496:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177184.844547:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177184.844551:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177184.844554:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177185.484466:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177185.484473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177185.484474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177185.484479:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177185.484482:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177185.484485:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.484487:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177185.484489:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.484491:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177185.868463:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177185.868468:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177185.868471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177185.868472:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177185.868474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177185.868475:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177185.868522:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177185.868531:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177185.868534:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177185.868567:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177185.868569:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177185.868571:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177185.868573:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177185.868576:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177185.868578:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.868580:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177185.868582:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.868584:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177185.996479:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177185.996488:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177185.996490:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177185.996493:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177185.996497:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177185.996503:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11860 00000400:00000200:3.0:1786177185.996511:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177185.996517:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177185.996520:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177185.996523:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996526:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177185.996528:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996530:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177185.996534:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177185.996536:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177185.996538:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177185.996541:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177185.996543:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177185.996545:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177185.996548:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996550:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177185.996552:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996554:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177185.996557:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177185.996565:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996568:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177185.996574:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177185.996577:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177185.996579:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.996581:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177185.996584:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177185.996592:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786177185.996595:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177185.996598:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177185.996600:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.996603:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177185.996605:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177185.996609:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177185.996615:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786177185.996621:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177185.996629:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177185.996638:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177185.996642:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177185.996646:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177185.996667:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177185.996670:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177185.996673:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177185.996674:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177185.996676:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177185.996678:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177185.996680:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11860 00000400:00000200:3.0:1786177185.996684:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177185.996686:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177185.996689:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177185.996692:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996694:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177185.996697:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996700:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177185.996703:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177185.996704:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1786177185.996705:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177185.996707:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177185.996710:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1786177185.996711:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00000001:1.0:1786177185.996715:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1786177185.996716:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177185.996720:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177185.996725:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177185.996726:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996729:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177185.996732:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996736:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177185.996741:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177185.996748:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177185.996751:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177185.996752:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.996753:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177185.996757:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177185.996759:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177185.996761:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177185.996762:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177185.996764:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177185.996767:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177185.996770:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235585920:192.168.202.132@tcp:250:kworker.0 00000100:00000001:3.0:1786177185.996771:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177185.996775:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1786177185.996776:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1786177185.996778:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1786177185.996779:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177185.996781:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177185.996783:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177185.996787:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 00000100:00000001:3.0:1786177185.996787:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177185.996791:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177185.996793:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786177185.996796:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177185.996797:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:3.0:1786177185.996799:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1786177185.996802:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 02000000:00000001:3.0:1786177185.996802:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.996806:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177185.996808:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1786177185.996810:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 02000000:00000001:3.0:1786177185.996810:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177185.996817:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000200:1.0:1786177185.996818:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235585920, portal 25 00000100:00000001:3.0:1786177185.996820:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.996821:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177185.996824:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280810880 : -110371428740736 : ffff9b9e26325180) 00000100:00100000:3.0:1786177185.996829:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1786177185.996830:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177185.996854:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177185.996855:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1786177185.996858:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177185.996862:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177185.996864:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1786177185.996867:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177185.996871:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235585920, offset 0 00000100:00000001:3.0:1786177185.996879:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177185.996883:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177185.996883:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177185.996888:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177185.996889:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177185.996890:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177185.996892:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177185.996894:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:3.0:1786177185.996895:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11860 00000400:00000200:3.0:1786177185.996900:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177185.996903:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177185.996904:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177185.996909:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000040:3.0:1786177185.996909:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996912:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177185.996916:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996917:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177185.996919:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1786177185.996919:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177185.996921:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177185.996923:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177185.996924:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:1.0:1786177185.996927:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000001:3.0:1786177185.996927:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177185.996931:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177185.996935:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1786177185.996935:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996940:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177185.996943:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000040:3.0:1786177185.996944:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996948:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177185.996950:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:1.0:1786177185.996953:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426644:426644:256:4294967295] 192.168.202.132@tcp LPNI seq info [426606:426606:8:4294967295] 00000100:00080000:3.0:1786177185.996954:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:1.0:1786177185.996959:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177185.996959:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177185.996960:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177185.996964:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177185.996965:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.996967:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177185.996969:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177185.996973:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.996976:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177185.996977:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000010:3.0:1786177185.996978:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786177185.996983:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177185.996986:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177185.996988:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177185.996990:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177185.996990:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177185.996992:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177185.996994:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177185.996998:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000800:00000010:1.0:1786177185.996999:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000001:3.0:1786177185.997000:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177185.997006:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1786177185.997011:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177185.997015:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177185.997017:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177185.997018:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000040:3.0:1786177185.997021:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1786177185.997029:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177185.997031:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177185.997034:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1786177185.997036:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00080000:3.0:1786177185.997037:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177185.997039:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177185.997042:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177185.997043:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:1.0:1786177185.997045:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177185.997048:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00080000:3.0:1786177185.997048:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11860 00000400:00000200:3.0:1786177185.997053:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177185.997055:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1786177185.997056:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177185.997060:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177185.997063:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1786177185.997065:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.997067:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177185.997068:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000100:00000040:3.0:1786177185.997069:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177185.997071:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:3.0:1786177185.997072:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177185.997075:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177185.997076:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:1.0:1786177185.997077:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1786177185.997078:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177185.997081:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000200:1.0:1786177185.997083:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177185.997083:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177185.997088:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177185.997094:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.997097:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177185.997099:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.997103:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177185.997104:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:1.0:1786177185.997105:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1786177185.997106:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177185.997110:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177185.997112:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1786177185.997113:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177185.997114:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177185.997117:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.997120:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177185.997122:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177185.997125:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.997129:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.997133:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177185.997133:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786177185.997137:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177185.997138:0:2392:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1786177185.997140:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177185.997142:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.997143:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177185.997145:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177185.997146:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.997147:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177185.997149:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786177185.997152:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177185.997155:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:3.0:1786177185.997157:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177185.997161:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306536704 00000100:00000001:3.0:1786177185.997164:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177185.997167:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177185.997169:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1786177185.997170:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62b40. 00000100:00000001:3.0:1786177185.997177:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177185.997179:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177185.997180:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1786177185.997181:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177185.997183:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177185.997185:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177185.997186:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177185.997188:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11860 00000100:00000001:1.0:1786177185.997189:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306536704 00000400:00000200:3.0:1786177185.997191:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000010:1.0:1786177185.997193:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62660. 00000400:00000200:3.0:1786177185.997196:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177185.997197:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1786177185.997198:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177185.997200:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1786177185.997203:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177185.997204:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.997206:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177185.997207:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1786177185.997210:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177185.997212:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00100000:1.0:1786177185.997214:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177185/real 1786177185] req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177185.997215:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177185.997217:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177185.997220:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177185.997222:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177185.997226:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177185.997228:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000001:1.0:1786177185.997229:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177185.997232:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177185.997235:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000100:00000040:3.0:1786177185.997235:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177185.997240:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000100:00000001:3.0:1786177185.997240:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177185.997243:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177185.997244:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:3.0:1786177185.997245:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177185.997247:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:3.0:1786177185.997249:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000200:1.0:1786177185.997252:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177185.997253:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177185.997258:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177185.997261:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177185.997263:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1786177185.997267:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177185.997267:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.997269:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177185.997271:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177185.997275:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.997278:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.997280:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177185.997282:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1786177185.997282:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000100:00000001:1.0:1786177185.997284:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177185.997286:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1786177185.997286:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177185.997291:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1786177185.997292:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177185.997293:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177185.997297:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177185.997299:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177185.997301:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.997305:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1786177185.997307:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000040:1.0:1786177185.997309:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177185.997311:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177185.997318:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177185.997319:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177185.997325:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177185.997327:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177185.997329:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1786177185.997330:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.997332:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177185.997334:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997337:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177185.997340:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177185.997342:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177185.997346:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11870) 00000100:00080000:1.0:1786177185.997347:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177185.997353:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177185.997381:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177185.997385:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177185.997391:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235585920:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177185.997394:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.997395:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177185.997401:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325180 x1872930235585920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997405:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177185.997407:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177185.997409:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177185.997411:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177185.997412:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177185.997413:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177185.997415:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177185.997416:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9400. 02000000:00000001:1.0:1786177185.997419:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177185.997420:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177185.997421:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325180. 00000100:00000001:1.0:1786177185.997423:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177185.997424:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.997425:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.997430:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177185.997432:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.997433:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177185.997434:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177185.997435:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177185.997437:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997456:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177185.997457:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177185.997459:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177185.997461:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177185.997464:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177185.997467:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586048:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177185.997469:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177185.997471:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177185.997473:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177185.997475:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177185.997479:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177185.997480:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177185.997484:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177185.997486:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177185.997489:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177185.997492:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235586048, portal 10 00000100:00000001:1.0:1786177185.997494:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177185.997495:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818048 : -110371428733568 : ffff9b9e26326d80) 00000100:00000040:1.0:1786177185.997500:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997505:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177185.997508:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177185.997510:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177185.997511:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235586048, offset 0 00000400:00000200:1.0:1786177185.997515:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177185.997518:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177185.997522:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177185.997525:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177185.997527:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177185.997529:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177185.997531:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177185.997533:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426645:426645:256:4294967295] 192.168.202.132@tcp LPNI seq info [426607:426607:8:4294967295] 00000400:00000200:1.0:1786177185.997537:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177185.997543:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177185.997546:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177185.997550:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177185.997554:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177185.997556:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786177185.997560:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177185.997562:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1786177185.997565:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177185.997566:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177185.997569:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177185.997571:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177185.997574:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177185.997575:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177185.997577:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177185.997580:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997587:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177185.997590:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997594:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.997596:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177185.997599:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177185.997602:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177185.997605:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177185.997606:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306532868 00000100:00000001:1.0:1786177185.997608:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306536704 00000100:00000001:1.0:1786177185.997610:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306532868 00000100:00000001:1.0:1786177185.997612:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306536704 00000100:00000001:1.0:1786177185.997614:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177185.997615:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.997617:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177185.997620:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177185/real 1786177185] req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177185.997633:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177185.997634:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177185.997636:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177185.997638:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177185.997640:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177185.997642:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177185.997644:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177185.997649:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997653:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177185.997654:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.997655:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177185.997656:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177185.997658:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997661:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177185.997664:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997667:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177185.997669:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.997670:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177185.997670:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.997671:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177185.997673:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177185.997674:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177185.997676:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177185.997678:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177185.997683:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177185.997690:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586048:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177185.997693:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177185.997696:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997702:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177185.997704:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177185.997706:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177185.997707:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177185.997709:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177185.997713:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586176:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177185.997716:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177185.997718:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177185.997719:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177185.997721:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177185.997724:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177185.997726:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177185.997728:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177185.997730:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177185.997732:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177185.997734:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235586176, portal 10 00000100:00000001:1.0:1786177185.997735:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177185.997736:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280820736 : -110371428730880 : ffff9b9e26327800) 00000100:00000040:1.0:1786177185.997740:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997743:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177185.997745:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177185.997746:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177185.997747:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235586176, offset 0 00000400:00000200:1.0:1786177185.997750:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177185.997752:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177185.997755:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177185.997757:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177185.997758:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177185.997760:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177185.997761:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177185.997763:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426646:426646:256:4294967295] 192.168.202.132@tcp LPNI seq info [426608:426608:8:4294967295] 00000400:00000200:1.0:1786177185.997766:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177185.997769:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177185.997771:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177185.997773:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177185.997776:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177185.997777:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786177185.997780:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177185.997781:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1786177185.997784:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177185.997785:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177185.997787:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177185.997789:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177185.997791:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177185.997792:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177185.997793:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177185.997795:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997799:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177185.997801:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997805:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.997806:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177185.997808:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177185.997810:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177185.997812:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177185.997813:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306532868 00000100:00000001:1.0:1786177185.997815:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306536704 00000100:00000001:1.0:1786177185.997816:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306532868 00000100:00000001:1.0:1786177185.997817:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306536704 00000100:00000001:1.0:1786177185.997818:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177185.997819:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.997820:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177185.997822:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177185/real 1786177185] req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177185.997826:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177185.997827:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177185.997828:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177185.997829:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177185.997830:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177185.997831:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177185.997833:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177185.997837:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997841:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177185.997842:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.997843:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177185.997844:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177185.997846:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997849:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177185.997851:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997855:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177185.997857:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.997859:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177185.997860:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.997862:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177185.997863:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177185.997866:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177185.997868:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177185.997870:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177185.997874:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177185.997882:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586176:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177185.997885:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177185.997888:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997895:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177185.997896:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177185.997899:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177185.997900:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177185.997902:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177185.997904:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586304:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177185.997906:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177185.997907:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177185.997908:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177185.997909:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177185.997911:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177185.997913:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177185.997915:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177185.997916:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177185.997918:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177185.997920:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235586304, portal 4 00000100:00000001:1.0:1786177185.997922:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177185.997923:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280814464 : -110371428737152 : ffff9b9e26325f80) 00000100:00000040:1.0:1786177185.997927:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997931:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177185.997932:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177185.997933:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177185.997934:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235586304, offset 0 00000400:00000200:1.0:1786177185.997936:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177185.997939:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177185.997941:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177185.997942:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177185.997944:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177185.997945:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177185.997947:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177185.997949:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426647:426647:256:4294967295] 192.168.202.132@tcp LPNI seq info [426609:426609:8:4294967295] 00000400:00000200:1.0:1786177185.997951:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177185.997955:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177185.997957:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177185.997959:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177185.997961:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177185.997963:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786177185.997965:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177185.997966:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1786177185.997969:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177185.997970:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177185.997972:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177185.997974:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177185.997975:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177185.997976:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177185.997978:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177185.997979:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997988:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177185.997989:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.997993:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.997994:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177185.997996:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177185.997997:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177185.997999:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177185.998000:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306532868 00000100:00000001:1.0:1786177185.998001:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306536704 00000100:00000001:1.0:1786177185.998003:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306532868 00000100:00000001:1.0:1786177185.998004:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306536704 00000100:00000001:1.0:1786177185.998005:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177185.998006:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.998007:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177185.998009:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177185/real 1786177185] req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177185.998013:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177185.998013:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177185.998014:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177185.998016:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177185.998017:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177185.998018:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177185.998019:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177185.998024:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998027:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177185.998028:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998030:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177185.998030:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177185.998032:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998036:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177185.998038:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998041:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177185.998043:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998043:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177185.998044:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998045:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177185.998046:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177185.998047:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177185.998049:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177185.998050:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177185.998052:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177185.998056:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586304:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177185.998058:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177185.998060:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998063:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177185.998065:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177185.998066:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177185.998068:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177185.998070:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177185.998073:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586432:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177185.998075:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177185.998077:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177185.998078:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177185.998080:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177185.998082:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177185.998084:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177185.998086:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177185.998089:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177185.998091:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177185.998094:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235586432, portal 4 00000100:00000001:1.0:1786177185.998095:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177185.998097:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280819840 : -110371428731776 : ffff9b9e26327480) 00000100:00000040:1.0:1786177185.998102:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998108:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177185.998110:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177185.998112:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177185.998114:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235586432, offset 0 00000400:00000200:1.0:1786177185.998118:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177185.998122:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177185.998125:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177185.998127:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177185.998130:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177185.998132:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177185.998134:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177185.998137:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426648:426648:256:4294967295] 192.168.202.132@tcp LPNI seq info [426610:426610:8:4294967295] 00000400:00000200:1.0:1786177185.998141:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177185.998147:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177185.998151:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177185.998154:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177185.998158:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177185.998159:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786177185.998162:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177185.998163:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1786177185.998166:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177185.998167:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177185.998169:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177185.998171:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177185.998173:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177185.998174:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177185.998175:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177185.998178:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998182:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177185.998184:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177210 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998188:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.998189:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177185.998191:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177185.998193:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177185.998194:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177185.998196:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306532870 00000100:00000001:1.0:1786177185.998197:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306536704 00000100:00000001:1.0:1786177185.998198:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306532870 00000100:00000001:1.0:1786177185.998200:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306536704 00000100:00000001:1.0:1786177185.998201:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177185.998202:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.998203:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177185.998205:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177185/real 1786177185] req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177185.998209:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177185.998210:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177185.998211:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177185.998213:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177185.998214:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177185.998215:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177185.998217:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177185.998221:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998225:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177185.998225:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998226:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177185.998227:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177185.998229:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998232:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177185.998234:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998238:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177185.998239:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998240:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177185.998240:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998241:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177185.998242:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177185.998243:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177185.998245:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177185.998246:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177185.998248:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177185.998255:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586432:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177185.998258:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998259:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177185.998260:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326d80 x1872930235586048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998264:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177185.998265:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177185.998266:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177185.998267:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177185.998268:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177185.998268:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177185.998269:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177185.998270:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9000. 02000000:00000001:1.0:1786177185.998272:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177185.998272:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177185.998273:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326d80. 00000100:00000001:1.0:1786177185.998275:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177185.998276:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998277:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177185.998279:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327800 x1872930235586176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998282:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177185.998283:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177185.998283:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177185.998284:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177185.998285:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177185.998286:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177185.998287:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177185.998287:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa800. 02000000:00000001:1.0:1786177185.998288:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177185.998289:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177185.998290:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327800. 00000100:00000001:1.0:1786177185.998291:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177185.998292:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998293:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177185.998295:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325f80 x1872930235586304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998298:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177185.998299:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177185.998300:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:1.0:1786177185.998301:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177185.998301:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177185.998302:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177185.998303:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177185.998304:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa000. 02000000:00000001:1.0:1786177185.998304:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177185.998305:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177185.998306:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325f80. 00000100:00000001:1.0:1786177185.998307:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177185.998307:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998308:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177185.998310:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327480 x1872930235586432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177210 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177185.998313:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177185.998313:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177185.998314:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177185.998315:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177185.998315:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177185.998316:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177185.998317:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177185.998318:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:1.0:1786177185.998319:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177185.998319:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177185.998320:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327480. 00000100:00000001:1.0:1786177185.998322:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177185.998323:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998325:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177185.998328:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177185.998329:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177185.998331:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177185.998332:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177186.508456:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177186.508464:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177186.508466:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177186.508472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177186.508476:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177186.508481:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177186.508484:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177186.508486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177186.508491:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177186.892476:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177186.892482:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177186.892483:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177186.892487:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177186.892490:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177186.892493:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177186.892495:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177186.892496:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177186.892498:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177186.892500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177186.892504:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177186.892526:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177186.892534:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177186.892565:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177186.892566:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177186.892618:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177186.892622:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177186.892625:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177187.532481:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177187.532488:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177187.532490:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177187.532496:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177187.532500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177187.532503:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177187.532506:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177187.532508:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177187.532511:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177187.916466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177187.916472:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177187.916473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177187.916478:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177187.916481:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177187.916483:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177187.916484:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177187.916488:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177187.916489:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177187.916491:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177187.916495:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177187.916496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177187.916498:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177187.916503:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177187.916506:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177187.916565:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177187.916568:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177187.916573:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177188.556456:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177188.556462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177188.556464:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177188.556468:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177188.556472:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177188.556474:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177188.556476:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177188.556478:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177188.556480:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177188.940476:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177188.940480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177188.940483:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177188.940485:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177188.940487:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177188.940494:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177188.940494:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177188.940497:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177188.940499:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177188.940503:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177188.940504:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177188.940507:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177188.940508:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177188.940512:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177188.940516:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177188.940563:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177188.940566:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177188.940569:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177189.580485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177189.580493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177189.580495:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177189.580501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177189.580505:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177189.580509:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177189.580512:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177189.580514:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177189.580517:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177189.900470:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306536704, at 4306536768 00000100:00000040:1.0:1786177189.900470:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306536704, at 4306536768 00000100:00000010:0.0:1786177189.900486:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62b40. 00000100:00000010:1.0:1786177189.900492:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62660. 00000100:00000001:1.0:1786177189.964474:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177189.964478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177189.964482:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177189.964483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177189.964486:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177189.964492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177189.964492:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177189.964497:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177189.964498:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177189.964501:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177189.964503:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177189.964505:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177189.964507:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177189.964510:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177189.964512:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177189.964514:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177189.964516:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177189.964518:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177189.964522:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177189.964526:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177189.964527:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177189.964580:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177189.964582:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177189.964585:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177190.604457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177190.604465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177190.604467:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177190.604474:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177190.604477:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177190.604480:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177190.604482:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177190.604485:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177190.604488:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177190.988470:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177190.988475:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177190.988477:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177190.988479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177190.988481:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177190.988487:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177190.988488:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177190.988491:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177190.988493:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177190.988496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177190.988497:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177190.988500:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177190.988502:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177190.988506:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177190.988509:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177190.988562:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177190.988566:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177190.988569:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177191.116439:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177191.116447:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177191.116449:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177191.116452:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177191.116455:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177191.116461:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11865 00000400:00000200:3.0:1786177191.116467:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177191.116472:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177191.116474:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177191.116476:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116478:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177191.116480:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116482:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177191.116484:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177191.116486:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177191.116488:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177191.116490:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177191.116492:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177191.116493:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177191.116495:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116496:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177191.116497:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116498:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177191.116500:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177191.116502:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116503:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177191.116505:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177191.116507:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177191.116509:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.116510:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177191.116512:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177191.116519:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786177191.116522:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177191.116525:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177191.116526:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.116528:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177191.116530:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177191.116533:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177191.116538:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786177191.116543:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177191.116549:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177191.116556:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177191.116560:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177191.116562:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177191.116578:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177191.116580:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177191.116582:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177191.116583:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177191.116584:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177191.116586:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177191.116587:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11865 00000400:00000200:3.0:1786177191.116589:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177191.116591:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177191.116592:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177191.116594:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116595:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177191.116597:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116598:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177191.116600:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177191.116601:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177191.116602:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177191.116604:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177191.116606:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177191.116607:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177191.116609:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116610:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177191.116611:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1786177191.116612:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116615:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177191.116616:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1786177191.116617:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:1.0:1786177191.116620:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1786177191.116621:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116624:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177191.116625:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1786177191.116629:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177191.116630:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177191.116634:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.116637:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177191.116641:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1786177191.116642:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1786177191.116644:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1786177191.116645:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.116647:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177191.116648:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:3.0:1786177191.116650:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 02000000:00000001:1.0:1786177191.116651:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1786177191.116653:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177191.116655:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177191.116657:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1786177191.116659:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235586560:192.168.202.132@tcp:250:kworker.0 02000000:00000001:3.0:1786177191.116660:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.116663:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177191.116665:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1786177191.116666:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177191.116669:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177191.116670:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177191.116672:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177191.116672:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 02000000:00000001:1.0:1786177191.116677:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1786177191.116677:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177191.116681:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 00000100:00100000:3.0:1786177191.116683:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177191.116684:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.116692:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177191.116693:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177191.116695:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177191.116697:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000100:00000040:3.0:1786177191.116700:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1786177191.116704:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000001:3.0:1786177191.116708:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177191.116711:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235586560, portal 25 00000100:00080000:3.0:1786177191.116712:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177191.116717:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177191.116718:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1786177191.116718:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177191.116720:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177191.116721:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280816256 : -110371428735360 : ffff9b9e26326680) 00000100:00000001:3.0:1786177191.116722:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177191.116726:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11865 00000100:00000040:1.0:1786177191.116730:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1786177191.116732:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177191.116737:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177191.116741:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177191.116745:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1786177191.116745:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116747:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177191.116751:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177191.116751:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177191.116755:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000001:3.0:1786177191.116755:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177191.116757:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000200:1.0:1786177191.116759:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235586560, offset 0 00000020:00000040:3.0:1786177191.116759:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177191.116762:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177191.116764:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177191.116765:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177191.116768:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000020:00000001:3.0:1786177191.116768:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177191.116775:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177191.116783:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000040:3.0:1786177191.116783:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116786:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177191.116790:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1786177191.116791:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116794:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177191.116796:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:1.0:1786177191.116798:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00080000:3.0:1786177191.116801:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:1.0:1786177191.116804:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1786177191.116805:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177191.116809:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 02000000:00000001:3.0:1786177191.116810:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177191.116814:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1786177191.116817:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426649:426649:256:4294967295] 192.168.202.132@tcp LPNI seq info [426611:426611:8:4294967295] 02000000:00000001:3.0:1786177191.116817:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.116821:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177191.116824:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177191.116825:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000008:00000001:3.0:1786177191.116828:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.116830:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177191.116835:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000010:3.0:1786177191.116835:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000800:00000200:1.0:1786177191.116842:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177191.116842:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177191.116847:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177191.116849:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177191.116850:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000001:3.0:1786177191.116851:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177191.116853:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177191.116855:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177191.116858:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000800:00000200:1.0:1786177191.116859:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177191.116861:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177191.116863:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000100:00100000:3.0:1786177191.116867:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1786177191.116872:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177191.116877:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000100:00000001:3.0:1786177191.116880:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177191.116884:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1786177191.116885:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177191.116888:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00000040:3.0:1786177191.116889:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177191.116892:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177191.116899:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1786177191.116902:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177191.116904:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177191.116906:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00080000:3.0:1786177191.116906:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177191.116911:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1786177191.116912:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000200:1.0:1786177191.116916:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177191.116916:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177191.116919:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177191.116925:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177191.116929:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1786177191.116931:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11865 00000100:00000040:1.0:1786177191.116934:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1786177191.116939:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177191.116945:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177191.116947:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177191.116948:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177191.116949:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1786177191.116953:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177191.116954:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.116955:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177191.116959:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177191.116962:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177191.116963:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177191.116965:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306541824 00000020:00000001:3.0:1786177191.116966:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177191.116967:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177191.116970:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000010:1.0:1786177191.116972:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62660. 00000020:00000040:3.0:1786177191.116972:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177191.116976:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177191.116978:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177191.116980:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177191.116981:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1786177191.116982:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177191.116984:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.116986:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177191.116988:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306541824 00000020:00000040:3.0:1786177191.116988:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000010:1.0:1786177191.116992:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62a80. 00000100:00080000:3.0:1786177191.116993:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177191.116998:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1786177191.116998:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177191.117001:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177191.117003:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177191.117004:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177191.117005:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177191.117008:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117009:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1786177191.117010:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177191.117013:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1786177191.117014:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177191/real 1786177191] req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1786177191.117015:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.117019:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177191.117024:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000400:00000001:1.0:1786177191.117025:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:3.0:1786177191.117027:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000400:00000001:1.0:1786177191.117028:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177191.117031:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 02000000:00000001:3.0:1786177191.117032:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1786177191.117034:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 02000000:00000001:3.0:1786177191.117034:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.117036:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:1.0:1786177191.117037:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 02000000:00000001:3.0:1786177191.117038:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177191.117040:0:2392:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1786177191.117040:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177191.117044:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1786177191.117046:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786177191.117051:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177191.117055:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786177191.117057:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177191.117070:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1786177191.117072:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177191.117073:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177191.117076:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117077:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177191.117079:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177191.117083:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177191.117084:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117111:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177191.117112:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177191.117114:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177191.117117:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177191.117124:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177191.117126:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177191.117127:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177191.117129:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177191.117130:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117133:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1786177191.117133:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177191.117136:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117138:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1786177191.117139:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11865 00000100:00000001:1.0:1786177191.117140:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177191.117144:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:3.0:1786177191.117148:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177191.117150:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177191.117153:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:3.0:1786177191.117155:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000040:1.0:1786177191.117158:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177191.117160:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177191.117168:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00100000:1.0:1786177191.117171:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235586560:192.168.202.132@tcp:250:kworker.0 00000100:00000001:3.0:1786177191.117173:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177191.117175:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177191.117178:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177191.117179:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1786177191.117182:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:1.0:1786177191.117184:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326680 x1872930235586560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177191.117185:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177191.117187:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177191.117190:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177191.117192:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177191.117196:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177191.117197:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1786177191.117198:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 02000000:00000001:1.0:1786177191.117199:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177191.117203:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 00000100:00000040:3.0:1786177191.117203:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177191.117205:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177191.117207:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1786177191.117209:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000020:00000001:1.0:1786177191.117210:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:3.0:1786177191.117212:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177191.117213:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177191.117216:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000020:00000040:3.0:1786177191.117216:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 02000000:00000010:1.0:1786177191.117220:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 00000100:00080000:3.0:1786177191.117220:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:1.0:1786177191.117224:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1786177191.117224:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177191.117226:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177191.117227:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177191.117229:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326680. 02000000:00000001:3.0:1786177191.117229:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177191.117232:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1786177191.117232:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117234:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177191.117235:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177191.117237:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1786177191.117238:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177191.117241:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177191.117242:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117244:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:3.0:1786177191.117245:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000100:00000001:1.0:1786177191.117248:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1786177191.117248:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177191.117251:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1786177191.117252:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177191.117253:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1786177191.117254:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117256:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177191.117256:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177191.117259:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1786177191.117262:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177191.117262:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177191.117266:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177191.117271:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117272:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177191.117274:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177191.117277:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1786177191.117277:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177191.117279:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177191.117283:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177191.117289:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586688:192.168.202.132@tcp:38:kworker.0 00000100:00000001:3.0:1786177191.117289:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177191.117292:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117294:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177191.117297:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1786177191.117298:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177191.117299:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177191.117301:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177191.117303:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177191.117305:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177191.117308:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177191.117312:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000100:00000001:3.0:1786177191.117317:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177191.117319:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000040:3.0:1786177191.117321:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11875) 00000100:00000200:1.0:1786177191.117323:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235586688, portal 10 00000100:00000001:1.0:1786177191.117327:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177191.117329:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818944 : -110371428732672 : ffff9b9e26327100) 00000100:00000040:1.0:1786177191.117336:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117343:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177191.117346:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177191.117349:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177191.117351:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235586688, offset 0 00000400:00000200:1.0:1786177191.117355:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177191.117376:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177191.117379:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177191.117381:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177191.117383:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177191.117384:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177191.117386:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177191.117388:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426650:426650:256:4294967295] 192.168.202.132@tcp LPNI seq info [426612:426612:8:4294967295] 00000400:00000200:1.0:1786177191.117401:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177191.117406:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177191.117409:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177191.117411:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177191.117414:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177191.117416:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786177191.117418:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177191.117420:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1786177191.117422:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177191.117423:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177191.117425:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177191.117427:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177191.117429:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177191.117430:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177191.117431:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177191.117434:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117447:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177191.117450:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117457:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117458:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177191.117462:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177191.117465:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177191.117468:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177191.117470:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306537988 00000100:00000001:1.0:1786177191.117472:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306541824 00000100:00000001:1.0:1786177191.117474:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306537988 00000100:00000001:1.0:1786177191.117476:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306541824 00000100:00000001:1.0:1786177191.117478:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177191.117481:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117482:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177191.117485:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177191/real 1786177191] req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177191.117490:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177191.117491:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177191.117492:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177191.117493:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177191.117494:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177191.117496:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177191.117498:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177191.117502:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117506:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177191.117507:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117508:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177191.117509:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177191.117511:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117515:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177191.117517:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117521:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177191.117522:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117523:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177191.117524:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117525:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177191.117526:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177191.117527:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177191.117529:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177191.117530:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177191.117532:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177191.117536:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586688:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177191.117539:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177191.117540:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117544:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177191.117545:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177191.117546:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177191.117547:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177191.117549:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177191.117550:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586816:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177191.117552:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177191.117554:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177191.117555:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177191.117556:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177191.117558:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177191.117559:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177191.117561:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177191.117563:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177191.117564:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177191.117566:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235586816, portal 10 00000100:00000001:1.0:1786177191.117567:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177191.117568:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280807296 : -110371428744320 : ffff9b9e26324380) 00000100:00000040:1.0:1786177191.117572:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117575:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177191.117577:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177191.117578:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177191.117579:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235586816, offset 0 00000400:00000200:1.0:1786177191.117581:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177191.117584:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177191.117586:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177191.117588:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177191.117589:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177191.117590:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177191.117592:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177191.117594:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426651:426651:256:4294967295] 192.168.202.132@tcp LPNI seq info [426613:426613:8:4294967295] 00000400:00000200:1.0:1786177191.117596:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177191.117600:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177191.117602:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177191.117604:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177191.117606:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177191.117608:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786177191.117610:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177191.117612:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1786177191.117614:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177191.117615:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177191.117617:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177191.117620:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177191.117621:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177191.117622:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177191.117624:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177191.117626:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117630:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177191.117632:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117636:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117637:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177191.117638:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177191.117640:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177191.117642:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177191.117644:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306537988 00000100:00000001:1.0:1786177191.117645:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306541824 00000100:00000001:1.0:1786177191.117646:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306537988 00000100:00000001:1.0:1786177191.117648:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306541824 00000100:00000001:1.0:1786177191.117649:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177191.117650:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117651:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177191.117652:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177191/real 1786177191] req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177191.117656:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177191.117658:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177191.117659:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177191.117660:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177191.117661:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177191.117662:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177191.117664:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177191.117668:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117671:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177191.117672:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117673:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177191.117673:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177191.117675:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117678:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177191.117680:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117684:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177191.117685:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117687:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177191.117687:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117688:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177191.117689:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177191.117691:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177191.117692:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177191.117693:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177191.117695:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177191.117703:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586816:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177191.117705:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177191.117707:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117710:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177191.117711:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177191.117712:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177191.117713:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177191.117714:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177191.117715:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586944:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177191.117717:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177191.117718:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177191.117718:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177191.117719:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177191.117721:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177191.117722:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177191.117724:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177191.117725:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177191.117727:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177191.117728:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235586944, portal 4 00000100:00000001:1.0:1786177191.117730:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177191.117730:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280819840 : -110371428731776 : ffff9b9e26327480) 00000100:00000040:1.0:1786177191.117733:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117737:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177191.117738:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177191.117739:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177191.117740:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235586944, offset 0 00000400:00000200:1.0:1786177191.117742:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177191.117744:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177191.117746:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177191.117748:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177191.117749:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177191.117750:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177191.117752:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177191.117754:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426652:426652:256:4294967295] 192.168.202.132@tcp LPNI seq info [426614:426614:8:4294967295] 00000400:00000200:1.0:1786177191.117756:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177191.117759:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177191.117761:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177191.117763:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177191.117765:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177191.117766:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786177191.117768:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177191.117769:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1786177191.117772:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177191.117773:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177191.117775:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177191.117777:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177191.117778:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177191.117779:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177191.117780:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177191.117782:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117786:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177191.117788:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117791:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117792:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177191.117793:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177191.117795:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177191.117796:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177191.117797:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306537988 00000100:00000001:1.0:1786177191.117799:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306541824 00000100:00000001:1.0:1786177191.117800:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306537988 00000100:00000001:1.0:1786177191.117801:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306541824 00000100:00000001:1.0:1786177191.117802:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177191.117803:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117804:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177191.117806:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177191/real 1786177191] req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177191.117810:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177191.117810:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177191.117811:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177191.117812:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177191.117813:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177191.117814:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177191.117816:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177191.117820:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117824:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177191.117824:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117825:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177191.117826:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177191.117828:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117831:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177191.117833:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117836:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177191.117838:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117839:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177191.117840:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117841:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177191.117842:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177191.117843:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177191.117844:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177191.117846:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177191.117847:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177191.117851:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235586944:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177191.117853:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117854:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177191.117856:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327100 x1872930235586688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117859:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177191.117860:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177191.117861:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177191.117862:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177191.117863:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177191.117864:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177191.117865:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177191.117866:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:1.0:1786177191.117867:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177191.117868:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177191.117869:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327100. 00000100:00000001:1.0:1786177191.117870:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177191.117871:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117872:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177191.117874:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324380 x1872930235586816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117881:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177191.117882:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177191.117883:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177191.117884:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177191.117885:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177191.117886:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177191.117887:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177191.117887:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:1.0:1786177191.117888:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177191.117889:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177191.117890:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324380. 00000100:00000001:1.0:1786177191.117891:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177191.117891:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117892:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177191.117894:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327480 x1872930235586944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117897:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177191.117897:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177191.117898:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177191.117899:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177191.117900:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177191.117900:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177191.117901:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177191.117902:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:1.0:1786177191.117903:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177191.117903:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177191.117904:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327480. 00000100:00000001:1.0:1786177191.117905:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177191.117906:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117907:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.117909:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177191.117909:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.117910:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177191.117911:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177191.117912:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177191.117914:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117918:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177191.117919:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177191.117920:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177191.117921:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177191.117922:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177191.117923:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587072:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177191.117925:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177191.117926:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177191.117926:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177191.117927:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177191.117929:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177191.117930:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177191.117931:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177191.117933:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177191.117934:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177191.117936:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235587072, portal 4 00000100:00000001:1.0:1786177191.117937:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177191.117938:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280814464 : -110371428737152 : ffff9b9e26325f80) 00000100:00000040:1.0:1786177191.117941:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117945:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177191.117946:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177191.117948:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177191.117949:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235587072, offset 0 00000400:00000200:1.0:1786177191.117951:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177191.117952:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177191.117954:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177191.117956:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177191.117957:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177191.117958:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177191.117960:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177191.117962:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426653:426653:256:4294967295] 192.168.202.132@tcp LPNI seq info [426615:426615:8:4294967295] 00000400:00000200:1.0:1786177191.117964:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177191.117967:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177191.117970:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177191.117971:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177191.117973:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177191.117975:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1786177191.117977:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177191.117978:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1786177191.117981:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177191.117982:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177191.117983:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177191.117985:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177191.117987:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177191.117988:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177191.117989:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177191.117990:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.117995:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177191.117996:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177216 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.118000:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.118001:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177191.118002:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177191.118004:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177191.118006:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177191.118007:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306537990 00000100:00000001:1.0:1786177191.118008:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306541824 00000100:00000001:1.0:1786177191.118009:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306537990 00000100:00000001:1.0:1786177191.118011:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306541824 00000100:00000001:1.0:1786177191.118012:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177191.118013:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.118014:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177191.118016:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177191/real 1786177191] req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177191.118021:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177191.118021:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177191.118022:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177191.118023:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177191.118024:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177191.118025:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177191.118027:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177191.118031:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.118034:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177191.118035:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.118036:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177191.118037:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177191.118039:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.118042:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177191.118044:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.118047:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177191.118048:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.118049:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177191.118050:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.118051:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177191.118051:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177191.118053:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177191.118055:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177191.118056:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177191.118058:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177191.118062:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587072:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177191.118064:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.118065:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177191.118067:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325f80 x1872930235587072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177216 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177191.118070:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177191.118071:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177191.118072:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177191.118073:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177191.118073:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177191.118074:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177191.118075:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177191.118076:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:1.0:1786177191.118077:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177191.118077:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177191.118078:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325f80. 00000100:00000001:1.0:1786177191.118079:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177191.118080:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.118081:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.118082:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177191.118083:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.118084:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177191.118085:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177191.628458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177191.628465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177191.628467:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177191.628473:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177191.628478:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177191.628482:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.628486:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177191.628489:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177191.628494:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177192.012440:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177192.012447:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177192.012448:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177192.012452:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177192.012455:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177192.012455:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177192.012458:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177192.012460:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177192.012461:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177192.012465:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177192.012466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177192.012468:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177192.012470:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177192.012475:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177192.012477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177192.012527:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177192.012530:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177192.012533:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177192.652480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177192.652488:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177192.652491:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177192.652498:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177192.652502:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177192.652505:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177192.652508:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177192.652510:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177192.652513:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177193.036423:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177193.036430:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177193.036432:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177193.036436:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177193.036439:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177193.036442:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177193.036444:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177193.036445:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177193.036446:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177193.036450:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177193.036450:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177193.036453:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177193.036457:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177193.036460:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177193.036462:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177193.036464:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177193.036465:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177193.036468:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177193.036469:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177193.036471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177193.036473:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177193.036474:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177193.036475:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177193.036476:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177193.036476:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177193.036477:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177193.036478:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177193.036519:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177193.036522:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177193.036524:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177193.676455:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177193.676463:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177193.676466:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177193.676472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177193.676476:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177193.676480:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177193.676483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177193.676486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177193.676490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177194.060442:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177194.060445:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177194.060451:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177194.060451:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177194.060454:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177194.060458:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177194.060461:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177194.060462:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177194.060465:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177194.060466:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177194.060470:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177194.060472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177194.060473:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177194.060477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177194.060478:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177194.060531:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177194.060534:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177194.060537:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177194.700481:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177194.700489:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177194.700491:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177194.700495:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177194.700499:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177194.700502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177194.700504:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177194.700506:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177194.700510:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177195.020496:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306541824, at 4306541888 00000100:00000040:1.0:1786177195.020505:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306541824, at 4306541888 00000100:00000010:0.0:1786177195.020511:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62660. 00000100:00000010:1.0:1786177195.020517:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62a80. 00000100:00000001:1.0:1786177195.084477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177195.084501:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177195.084503:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177195.084508:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177195.084512:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177195.084515:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177195.084518:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177195.084520:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177195.084532:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177195.084534:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177195.084538:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177195.084543:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177195.084545:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177195.084547:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177195.084548:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177195.084620:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177195.084624:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177195.084628:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177195.724464:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177195.724472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177195.724475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177195.724481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177195.724486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177195.724490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177195.724492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177195.724497:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177195.724501:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.108434:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177196.108438:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177196.108444:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.108444:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177196.108448:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177196.108451:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177196.108455:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177196.108455:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177196.108462:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177196.108464:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177196.108468:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.108468:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.108472:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177196.108474:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.108477:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177196.108521:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177196.108525:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177196.108527:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177196.236465:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177196.236474:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177196.236478:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177196.236482:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177196.236486:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177196.236493:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11870 00000400:00000200:3.0:1786177196.236502:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177196.236509:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177196.236512:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177196.236516:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236519:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177196.236522:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236525:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177196.236529:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177196.236531:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177196.236534:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177196.236537:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177196.236540:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177196.236543:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177196.236546:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236548:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177196.236550:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236552:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177196.236555:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177196.236559:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236562:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177196.236565:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177196.236569:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177196.236572:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177196.236574:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177196.236576:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177196.236586:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786177196.236590:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177196.236594:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177196.236597:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177196.236600:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177196.236602:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177196.236607:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177196.236614:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786177196.236620:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177196.236632:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177196.236643:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177196.236647:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177196.236653:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177196.236676:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177196.236678:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177196.236681:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177196.236682:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177196.236683:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177196.236685:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177196.236687:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11870 00000400:00000200:3.0:1786177196.236690:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177196.236692:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177196.236694:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177196.236696:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236697:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177196.236699:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236700:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177196.236702:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177196.236703:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177196.236704:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177196.236705:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177196.236707:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177196.236709:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177196.236711:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236713:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177196.236714:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236716:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177196.236717:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177196.236719:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236721:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177196.236722:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177196.236726:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177196.236728:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177196.236729:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177196.236732:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177196.236734:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.236736:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1786177196.236736:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177196.236739:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786177196.236742:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000100:00000001:1.0:1786177196.236743:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1786177196.236745:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177196.236747:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.236748:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177196.236750:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177196.236752:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177196.236754:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177196.236758:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000040:1.0:1786177196.236760:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177196.236762:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.236773:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177196.236777:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177196.236778:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177196.236784:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1786177196.236786:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1786177196.236788:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1786177196.236790:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177196.236796:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177196.236797:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177196.236805:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235587200:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177196.236817:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1786177196.236820:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177196.236823:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1786177196.236823:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1786177196.236828:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177196.236829:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177196.236831:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177196.236834:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1786177196.236835:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1786177196.236836:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177196.236840:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11870 00000400:00000200:3.0:1786177196.236842:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:1.0:1786177196.236843:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 00000400:00000200:3.0:1786177196.236845:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 02000000:00000001:1.0:1786177196.236849:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177196.236849:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177196.236855:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236856:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177196.236859:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177196.236861:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177196.236861:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177196.236865:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177196.236866:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000400:00000010:1.0:1786177196.236868:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000020:00000001:3.0:1786177196.236868:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177196.236870:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177196.236873:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177196.236875:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177196.236877:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177196.236878:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000001:3.0:1786177196.236879:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177196.236883:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236885:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177196.236886:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00000200:1.0:1786177196.236889:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235587200, portal 25 00000100:00080000:3.0:1786177196.236889:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177196.236893:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.236895:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1786177196.236896:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177196.236900:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280820736 : -110371428730880 : ffff9b9e26327800) 02000000:00000001:3.0:1786177196.236901:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177196.236907:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177196.236909:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177196.236912:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1786177196.236914:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1786177196.236917:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177196.236922:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177196.236930:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000100:00000001:1.0:1786177196.236931:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:3.0:1786177196.236936:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000040:1.0:1786177196.236937:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 02000000:00000001:3.0:1786177196.236944:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1786177196.236945:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 02000000:00000001:3.0:1786177196.236947:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177196.236949:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235587200, offset 0 00000100:00000001:3.0:1786177196.236952:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177196.236955:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177196.236957:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177196.236960:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 02000000:00000010:3.0:1786177196.236965:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786177196.236970:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177196.236972:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:3.0:1786177196.236979:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177196.236983:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177196.236992:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1786177196.236995:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177196.237000:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:3.0:1786177196.237000:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177196.237007:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1786177196.237009:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177196.237015:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177196.237025:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426654:426654:256:4294967295] 192.168.202.132@tcp LPNI seq info [426616:426616:8:4294967295] 00000100:00000001:3.0:1786177196.237027:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177196.237032:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177196.237038:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:3.0:1786177196.237040:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177196.237043:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177196.237045:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177196.237053:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177196.237055:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00080000:3.0:1786177196.237060:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11870 00000800:00000200:1.0:1786177196.237067:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000400:00000200:3.0:1786177196.237068:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000010:1.0:1786177196.237076:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000400:00000200:3.0:1786177196.237077:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177196.237083:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1786177196.237089:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786177196.237090:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.237095:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1786177196.237096:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00000040:3.0:1786177196.237099:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.237105:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177196.237110:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000200:1.1:1786177196.237110:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786177196.237112:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177196.237117:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:1.1:1786177196.237118:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000020:00000040:3.0:1786177196.237120:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177196.237126:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000100:1.0:1786177196.237129:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000020:00000001:3.0:1786177196.237131:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000010:1.0:1786177196.237134:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00000040:3.0:1786177196.237139:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.237143:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177196.237144:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1786177196.237147:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.237152:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177196.237153:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000020:00000040:3.0:1786177196.237157:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000400:00000200:1.0:1786177196.237162:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000100:00080000:3.0:1786177196.237163:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000010:1.0:1786177196.237167:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:3.0:1786177196.237169:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177196.237173:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177196.237175:0:2392:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1786177196.237177:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1786177196.237183:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177196.237183:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177196.237187:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177196.237191:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177196.237197:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177196.237201:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237202:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000010:3.0:1786177196.237207:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000100:00000040:1.0:1786177196.237208:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177196.237213:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177196.237220:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177196.237223:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177196.237227:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177196.237230:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177196.237234:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237237:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177196.237240:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:1.0:1786177196.237241:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1786177196.237248:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237255:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177196.237261:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177196.237265:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306546944 00000100:00100000:3.0:1786177196.237269:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:1.0:1786177196.237271:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62a80. 00000100:00000001:1.0:1786177196.237282:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1786177196.237284:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177196.237288:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237292:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306546944 00000100:00000040:3.0:1786177196.237296:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1786177196.237299:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62e40. 00000100:00000001:1.0:1786177196.237305:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177196.237311:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1786177196.237315:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237316:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237321:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1786177196.237321:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177196.237328:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1786177196.237330:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177196/real 1786177196] req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177196.237331:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177196.237335:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177196.237392:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177196.237399:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11870 00000400:00000001:1.0:1786177196.237405:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:3.0:1786177196.237408:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000001:1.0:1786177196.237409:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177196.237413:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:3.0:1786177196.237416:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177196.237419:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000100:00080000:3.0:1786177196.237421:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1786177196.237425:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000040:3.0:1786177196.237429:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177196.237431:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1786177196.237435:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1786177196.237441:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177196.237442:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.237449:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177196.237455:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177196.237458:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000200:1.0:1786177196.237461:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177196.237462:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177196.237466:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177196.237472:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177196.237477:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177196.237479:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177196.237484:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177196.237486:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177196.237488:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177196.237491:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177196.237493:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1786177196.237500:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177196.237500:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177196.237506:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177196.237512:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:1.0:1786177196.237519:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177196.237520:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177196.237526:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177196.237528:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177196.237529:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177196.237535:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177196.237541:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237543:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177196.237545:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177196.237550:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1786177196.237551:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177196.237554:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:3.0:1786177196.237556:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237558:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177196.237560:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237562:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177196.237566:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000010:3.0:1786177196.237566:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786177196.237571:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00080000:1.0:1786177196.237573:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1786177196.237577:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177196.237579:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177196.237581:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:3.0:1786177196.237582:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177196.237585:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1786177196.237586:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177196.237590:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177196.237595:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1786177196.237598:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177196.237604:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177196.237611:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235587200:192.168.202.132@tcp:250:kworker.0 00000100:00100000:3.0:1786177196.237615:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177196.237619:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.237625:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1786177196.237630:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1786177196.237633:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327800 x1872930235587200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177196.237635:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177196.237646:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.237649:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177196.237653:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177196.237657:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 00000100:00000001:3.0:1786177196.237660:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177196.237662:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177196.237666:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000040:3.0:1786177196.237667:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11880) 00000020:00000040:1.0:1786177196.237669:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177196.237673:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177196.237676:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa400. 02000000:00000001:1.0:1786177196.237681:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177196.237684:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177196.237687:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327800. 00000100:00000001:1.0:1786177196.237691:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177196.237693:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.237696:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.237703:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177196.237705:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237707:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177196.237709:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177196.237710:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177196.237715:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.237723:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177196.237725:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177196.237727:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177196.237729:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177196.237732:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177196.237735:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587328:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177196.237739:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177196.237741:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177196.237743:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177196.237745:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177196.237748:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177196.237750:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177196.237755:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177196.237758:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177196.237764:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177196.237767:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235587328, portal 10 00000100:00000001:1.0:1786177196.237769:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177196.237771:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818048 : -110371428733568 : ffff9b9e26326d80) 00000100:00000040:1.0:1786177196.237779:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.237787:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177196.237790:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177196.237793:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177196.237796:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235587328, offset 0 00000400:00000200:1.0:1786177196.237800:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177196.237805:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177196.237809:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177196.237812:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177196.237815:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177196.237817:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177196.237820:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177196.237823:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426655:426655:256:4294967295] 192.168.202.132@tcp LPNI seq info [426617:426617:8:4294967295] 00000400:00000200:1.0:1786177196.237827:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177196.237835:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177196.237840:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177196.237844:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177196.237849:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177196.237852:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786177196.237857:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177196.237860:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1786177196.237864:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177196.237866:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177196.237869:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177196.237873:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177196.237876:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177196.237878:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177196.237881:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177196.237885:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.237893:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177196.237896:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.237903:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237905:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177196.237909:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177196.237913:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177196.237916:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177196.237918:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306543108 00000100:00000001:1.0:1786177196.237921:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306546944 00000100:00000001:1.0:1786177196.237924:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306543108 00000100:00000001:1.0:1786177196.237926:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306546944 00000100:00000001:1.0:1786177196.237928:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177196.237930:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.237932:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177196.237937:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177196/real 1786177196] req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177196.237945:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177196.237946:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177196.237948:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177196.237951:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177196.237952:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177196.237955:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177196.237958:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177196.237967:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.237973:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177196.237975:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.237976:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177196.237978:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177196.237981:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.237988:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177196.237992:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238000:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177196.238002:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238004:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177196.238005:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238007:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177196.238010:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177196.238013:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177196.238016:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177196.238018:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177196.238022:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177196.238031:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587328:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177196.238035:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177196.238039:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238045:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177196.238047:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177196.238049:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177196.238051:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177196.238053:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177196.238056:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587456:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177196.238059:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177196.238061:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177196.238063:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177196.238064:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177196.238068:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177196.238070:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177196.238073:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177196.238076:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177196.238080:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177196.238082:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235587456, portal 10 00000100:00000001:1.0:1786177196.238084:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177196.238086:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280810880 : -110371428740736 : ffff9b9e26325180) 00000100:00000040:1.0:1786177196.238091:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238098:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177196.238100:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177196.238102:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177196.238105:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235587456, offset 0 00000400:00000200:1.0:1786177196.238110:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177196.238114:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177196.238117:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177196.238121:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177196.238123:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177196.238126:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177196.238128:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177196.238131:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426656:426656:256:4294967295] 192.168.202.132@tcp LPNI seq info [426618:426618:8:4294967295] 00000400:00000200:1.0:1786177196.238135:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177196.238141:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177196.238146:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177196.238149:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177196.238153:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177196.238156:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786177196.238160:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177196.238163:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1786177196.238167:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177196.238168:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177196.238171:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177196.238175:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177196.238179:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177196.238182:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177196.238185:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177196.238189:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238197:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177196.238201:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238207:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.238209:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177196.238212:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177196.238216:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177196.238219:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177196.238221:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306543108 00000100:00000001:1.0:1786177196.238224:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306546944 00000100:00000001:1.0:1786177196.238226:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306543108 00000100:00000001:1.0:1786177196.238229:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306546944 00000100:00000001:1.0:1786177196.238231:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177196.238233:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.238235:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177196.238240:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177196/real 1786177196] req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177196.238248:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177196.238249:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177196.238252:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177196.238254:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177196.238256:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177196.238258:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177196.238262:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177196.238271:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238279:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177196.238280:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238283:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177196.238284:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177196.238288:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238295:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177196.238300:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238307:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177196.238310:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238311:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177196.238313:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238315:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177196.238317:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177196.238319:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177196.238323:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177196.238325:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177196.238329:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177196.238337:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587456:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177196.238373:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177196.238377:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238385:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177196.238386:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177196.238389:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177196.238391:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177196.238394:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177196.238397:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587584:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177196.238400:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177196.238402:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177196.238404:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177196.238406:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177196.238409:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177196.238411:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177196.238415:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177196.238418:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177196.238421:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177196.238424:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235587584, portal 4 00000100:00000001:1.0:1786177196.238426:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177196.238428:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280813568 : -110371428738048 : ffff9b9e26325c00) 00000100:00000040:1.0:1786177196.238434:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238441:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177196.238444:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177196.238446:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177196.238448:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235587584, offset 0 00000400:00000200:1.0:1786177196.238452:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177196.238457:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177196.238460:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177196.238463:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177196.238466:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177196.238469:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177196.238471:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177196.238474:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426657:426657:256:4294967295] 192.168.202.132@tcp LPNI seq info [426619:426619:8:4294967295] 00000400:00000200:1.0:1786177196.238479:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177196.238485:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177196.238490:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177196.238493:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177196.238497:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177196.238500:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786177196.238504:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177196.238507:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1786177196.238511:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177196.238513:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177196.238516:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177196.238520:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177196.238523:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177196.238525:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177196.238528:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177196.238531:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238539:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177196.238542:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238549:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.238550:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177196.238553:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177196.238556:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177196.238559:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177196.238562:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306543109 00000100:00000001:1.0:1786177196.238564:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306546944 00000100:00000001:1.0:1786177196.238567:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306543109 00000100:00000001:1.0:1786177196.238569:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306546944 00000100:00000001:1.0:1786177196.238571:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177196.238573:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.238575:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177196.238579:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177196/real 1786177196] req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177196.238588:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177196.238589:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177196.238591:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177196.238593:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177196.238594:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177196.238596:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177196.238600:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177196.238608:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238615:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177196.238616:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238618:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177196.238620:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177196.238623:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238630:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177196.238635:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238642:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177196.238644:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238646:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177196.238648:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238649:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177196.238651:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177196.238654:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177196.238657:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177196.238660:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177196.238665:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177196.238673:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587584:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177196.238678:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177196.238681:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238688:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177196.238690:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177196.238692:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177196.238694:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177196.238696:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177196.238699:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587712:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177196.238702:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177196.238704:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177196.238706:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177196.238708:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177196.238711:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261adc00. 02000000:00000001:1.0:1786177196.238713:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177196.238717:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177196.238719:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177196.238723:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177196.238726:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235587712, portal 4 00000100:00000001:1.0:1786177196.238728:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177196.238729:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280815360 : -110371428736256 : ffff9b9e26326300) 00000100:00000040:1.0:1786177196.238736:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238742:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177196.238745:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177196.238747:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177196.238749:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235587712, offset 0 00000400:00000200:1.0:1786177196.238754:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177196.238758:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177196.238762:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177196.238764:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177196.238767:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177196.238770:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177196.238772:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177196.238776:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426658:426658:256:4294967295] 192.168.202.132@tcp LPNI seq info [426620:426620:8:4294967295] 00000400:00000200:1.0:1786177196.238780:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177196.238786:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177196.238791:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177196.238794:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177196.238799:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177196.238801:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1786177196.238805:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177196.238809:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1786177196.238813:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177196.238815:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177196.238818:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177196.238821:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177196.238825:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177196.238827:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177196.238829:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177196.238833:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238841:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177196.238844:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177221 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238851:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.238853:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177196.238856:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177196.238858:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177196.238862:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177196.238864:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306543111 00000100:00000001:1.0:1786177196.238867:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306546944 00000100:00000001:1.0:1786177196.238869:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306543111 00000100:00000001:1.0:1786177196.238872:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306546944 00000100:00000001:1.0:1786177196.238874:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177196.238876:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.238878:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177196.238882:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177196/real 1786177196] req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177196.238890:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177196.238892:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177196.238893:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177196.238896:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177196.238898:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177196.238900:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177196.238904:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177196.238913:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238920:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177196.238922:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238924:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177196.238925:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177196.238929:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238936:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177196.238940:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238946:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177196.238949:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238951:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177196.238952:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238954:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177196.238955:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177196.238958:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177196.238962:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177196.238964:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177196.238968:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177196.238976:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587712:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177196.238980:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.238982:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177196.238985:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326d80 x1872930235587328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.238992:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177196.238994:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177196.238996:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177196.238999:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177196.239000:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177196.239002:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177196.239004:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177196.239006:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab800. 02000000:00000001:1.0:1786177196.239009:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177196.239010:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177196.239011:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326d80. 00000100:00000001:1.0:1786177196.239015:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177196.239016:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.239018:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177196.239022:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325180 x1872930235587456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.239036:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177196.239037:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177196.239039:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177196.239041:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177196.239042:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177196.239044:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177196.239046:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177196.239048:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8c00. 02000000:00000001:1.0:1786177196.239050:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177196.239051:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177196.239053:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325180. 00000100:00000001:1.0:1786177196.239055:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177196.239056:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.239058:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177196.239062:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325c00 x1872930235587584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.239068:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177196.239070:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177196.239072:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177196.239074:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177196.239075:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177196.239076:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177196.239079:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177196.239080:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8400. 02000000:00000001:1.0:1786177196.239083:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177196.239084:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177196.239085:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325c00. 00000100:00000001:1.0:1786177196.239087:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177196.239089:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.239090:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177196.239094:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326300 x1872930235587712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177221 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177196.239101:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177196.239102:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177196.239105:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261adc00. 02000000:00000001:1.0:1786177196.239107:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177196.239108:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177196.239109:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177196.239111:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177196.239113:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:1.0:1786177196.239115:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177196.239116:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177196.239117:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326300. 00000100:00000001:1.0:1786177196.239120:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177196.239121:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.239123:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.239126:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177196.239127:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177196.239128:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177196.239129:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177196.684494:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:1.0:1786177196.684504:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:1.0:1786177196.684509:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:1.0:1786177196.684676:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1786177196.684692:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1786177196.684693:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1786177196.684698:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1786177196.684703:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.748462:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177196.748470:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177196.748472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177196.748476:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177196.748479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177196.748483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.748485:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177196.748488:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177196.748491:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177197.132452:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177197.132456:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177197.132459:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177197.132460:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177197.132462:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177197.132468:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177197.132469:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177197.132471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177197.132474:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177197.132476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177197.132479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177197.132479:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177197.132483:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177197.132484:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177197.132486:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177197.132538:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177197.132541:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177197.132544:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177197.772464:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177197.772473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177197.772475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177197.772481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177197.772486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177197.772490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177197.772493:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177197.772495:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177197.772498:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177198.156421:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177198.156428:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177198.156430:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177198.156431:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177198.156436:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177198.156437:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177198.156441:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177198.156442:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177198.156444:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177198.156446:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177198.156449:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177198.156450:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177198.156451:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177198.156460:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177198.156462:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177198.156510:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177198.156513:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177198.156516:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177198.796477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177198.796486:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177198.796489:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177198.796495:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177198.796500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177198.796506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177198.796509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177198.796512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177198.796523:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177199.180466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177199.180473:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177199.180474:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177199.180480:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177199.180490:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177199.180493:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177199.180495:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177199.180497:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177199.180499:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177199.180555:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177199.180558:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177199.180561:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177199.180562:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177199.180565:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177199.180566:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177199.180633:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177199.180639:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177199.180643:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177199.820533:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177199.820542:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177199.820544:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177199.820549:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177199.820554:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177199.820565:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177199.820568:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177199.820574:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177199.820577:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177200.140523:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306546944, at 4306547008 00000100:00000040:1.0:1786177200.140527:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306546944, at 4306547008 00000100:00000010:0.0:1786177200.140543:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62a80. 00000100:00000010:1.0:1786177200.140547:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62e40. 00000100:00000001:1.0:1786177200.204460:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177200.204469:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177200.204472:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177200.204479:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177200.204483:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177200.204486:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177200.204489:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177200.204491:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177200.204494:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177200.204516:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177200.204522:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177200.204526:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177200.204529:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177200.204531:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177200.204534:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177200.204536:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177200.204538:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177200.204540:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177200.204542:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177200.204544:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177200.204545:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177200.204605:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177200.204609:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177200.204612:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177200.844512:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177200.844521:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177200.844523:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177200.844530:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177200.844534:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177200.844538:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177200.844542:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177200.844546:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177200.844550:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.228517:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177201.228523:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177201.228528:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177201.228530:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.228532:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177201.228542:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177201.228543:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177201.228548:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.228550:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177201.228556:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177201.228557:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177201.228559:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.228564:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177201.228569:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.228573:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786177201.228651:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786177201.228657:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786177201.228663:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177201.356478:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177201.356487:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177201.356489:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177201.356493:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177201.356496:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177201.356504:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11875 00000400:00000200:2.0:1786177201.356513:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177201.356520:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177201.356523:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177201.356528:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.356532:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177201.356534:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.356537:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177201.356542:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177201.356544:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177201.356547:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177201.356550:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177201.356554:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177201.356557:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177201.356560:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.356562:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177201.356564:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.356567:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177201.356570:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177201.356574:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177201.356577:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177201.356579:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177201.356582:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177201.356584:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.356587:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177201.356589:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177201.356597:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786177201.356600:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177201.356603:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177201.356605:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.356607:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177201.356609:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177201.356612:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177201.356618:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177201.356630:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177201.356641:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177201.356651:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177201.356656:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177201.356661:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177201.356682:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177201.356686:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177201.356690:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177201.356692:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177201.356693:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177201.356697:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177201.356700:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11875 00000400:00000200:2.0:1786177201.356704:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177201.356707:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177201.356709:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177201.356712:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.356714:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177201.356715:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177201.356718:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177201.356725:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177201.356726:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177201.356730:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1786177201.356732:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177201.356735:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177201.356738:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1786177201.356741:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177201.356742:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177201.356748:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177201.356753:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177201.356761:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177201.356764:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177201.356766:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177201.356768:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177201.356771:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177201.356775:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1786177201.356776:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:1.0:1786177201.356780:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1786177201.356782:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 02000000:00000001:1.0:1786177201.356786:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177201.356790:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00100000:1.0:1786177201.356795:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235587840:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177201.356796:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177201.356800:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177201.356805:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177201.356808:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177201.356810:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177201.356812:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177201.356815:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.356817:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1786177201.356821:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1786177201.356823:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177201.356828:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177201.356830:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 00000100:00000001:2.0:1786177201.356830:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177201.356836:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177201.356839:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 00000020:00000040:2.0:1786177201.356844:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177201.356847:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177201.356849:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177201.356853:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.356855:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:1.0:1786177201.356856:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 02000000:00000001:2.0:1786177201.356857:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177201.356862:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177201.356866:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 02000000:00000010:2.0:1786177201.356868:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:2.0:1786177201.356874:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177201.356877:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235587840, portal 25 00000100:00100000:2.0:1786177201.356882:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177201.356883:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177201.356887:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620992 : -110371239930624 : ffff9b9e31735500) 00000100:00000001:2.0:1786177201.356895:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177201.356900:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177201.356904:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177201.356910:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177201.356914:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177201.356919:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177201.356929:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177201.356933:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1786177201.356934:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177201.356938:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1786177201.356940:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177201.356943:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:1.0:1786177201.356946:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00080000:2.0:1786177201.356951:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11875 00000100:00000200:1.0:1786177201.356953:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235587840, offset 0 00000400:00000200:2.0:1786177201.356960:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177201.356966:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177201.356967:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177201.356972:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177201.356976:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.356979:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177201.356981:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177201.356985:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.356988:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177201.356990:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177201.356991:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000400:00000200:1.0:1786177201.356995:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000001:2.0:1786177201.356995:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177201.356999:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177201.357002:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177201.357004:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177201.357005:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1786177201.357008:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.357010:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177201.357011:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.357012:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177201.357014:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177201.357016:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177201.357020:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177201.357022:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1786177201.357025:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 02000000:00000001:2.0:1786177201.357025:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177201.357030:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.357032:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1786177201.357035:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000008:00100000:2.0:1786177201.357035:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177201.357041:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177201.357044:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.357044:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177201.357047:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000020:00000040:2.0:1786177201.357052:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177201.357054:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426659:426659:256:4294967295] 192.168.202.132@tcp LPNI seq info [426621:426621:8:4294967295] 02000000:00000001:2.0:1786177201.357055:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177201.357057:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.357060:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177201.357062:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177201.357066:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177201.357070:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:2.0:1786177201.357072:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:2.0:1786177201.357076:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177201.357082:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177201.357095:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177201.357097:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177201.357100:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177201.357104:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177201.357106:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177201.357109:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177201.357113:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177201.357115:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177201.357118:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177201.357121:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177201.357126:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11875 00000800:00000200:1.0:1786177201.357127:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177201.357132:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177201.357136:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000010:1.0:1786177201.357137:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00080000:2.0:1786177201.357138:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177201.357144:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.357146:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177201.357150:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:1.0:1786177201.357152:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177201.357152:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177201.357156:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177201.357158:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177201.357160:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:1.0:1786177201.357161:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000020:00000040:2.0:1786177201.357162:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177201.357166:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177201.357169:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177201.357173:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:1.1:1786177201.357175:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177201.357176:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177201.357180:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.357184:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:1.1:1786177201.357184:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1786177201.357194:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000020:00000040:2.0:1786177201.357196:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000800:00000010:1.0:1786177201.357198:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000100:00080000:2.0:1786177201.357204:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:1.0:1786177201.357207:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1786177201.357208:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177201.357213:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1786177201.357217:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1786177201.357218:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177201.357223:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177201.357226:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000100:00000001:2.0:1786177201.357226:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:1.0:1786177201.357231:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000008:00100000:2.0:1786177201.357236:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177201.357237:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:2.0:1786177201.357240:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.357243:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177201.357246:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1786177201.357247:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000020:00000040:2.0:1786177201.357252:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177201.357256:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177201.357259:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.357262:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177201.357264:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177201.357267:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.357270:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:2.0:1786177201.357271:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:2.0:1786177201.357276:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177201.357278:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177201.357283:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177201.357310:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177201.357313:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.357316:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177201.357317:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357321:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177201.357330:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.357331:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177201.357337:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177201.357340:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:2.0:1786177201.357344:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177201.357346:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306552064 00000100:00000001:2.0:1786177201.357348:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177201.357351:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:1.0:1786177201.357354:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62e40. 00000100:00000001:2.0:1786177201.357355:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177201.357389:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177201.357397:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306552065 00000100:00000010:1.0:1786177201.357402:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62960. 00000100:00080000:2.0:1786177201.357407:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11875 00000100:00000001:1.0:1786177201.357408:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177201.357416:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1786177201.357416:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177201.357422:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177201.357423:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177201.357425:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177201.357429:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1786177201.357430:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.357434:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177201.357436:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.357439:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00100000:1.0:1786177201.357441:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177201/real 1786177201] req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177201.357442:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177201.357444:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177201.357446:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177201.357449:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177201.357453:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177201.357456:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177201.357465:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000001:1.0:1786177201.357466:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1786177201.357467:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1786177201.357470:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:2.0:1786177201.357471:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177201.357475:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:1.0:1786177201.357476:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000020:00000040:2.0:1786177201.357477:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000400:00000200:1.0:1786177201.357482:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000100:00080000:2.0:1786177201.357482:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000010:1.0:1786177201.357488:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:2.0:1786177201.357488:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177201.357492:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177201.357495:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177201.357496:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.357498:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177201.357499:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177201.357500:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.357501:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.357505:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000010:2.0:1786177201.357506:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 00000020:00000040:2.0:1786177201.357510:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177201.357513:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1786177201.357514:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177201.357515:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177201.357519:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177201.357522:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177201.357524:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177201.357529:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:2.0:1786177201.357534:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177201.357538:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177201.357542:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177201.357567:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177201.357569:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177201.357570:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.357575:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177201.357576:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357582:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177201.357586:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177201.357587:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177201.357595:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11885) 00000100:00100000:1.0:1786177201.357599:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357609:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177201.357615:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357623:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177201.357626:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.357630:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177201.357631:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.357634:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177201.357636:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177201.357639:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177201.357643:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177201.357646:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177201.357651:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177201.357660:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235587840:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177201.357666:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.357668:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177201.357672:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735500 x1872930235587840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357679:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177201.357681:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177201.357684:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177201.357687:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177201.357689:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177201.357691:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177201.357693:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177201.357695:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:1.0:1786177201.357698:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177201.357700:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177201.357701:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735500. 00000100:00000001:1.0:1786177201.357704:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177201.357705:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.357708:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.357715:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177201.357717:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.357719:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177201.357721:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177201.357722:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177201.357727:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357734:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177201.357735:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177201.357738:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177201.357740:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177201.357742:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177201.357745:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587968:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177201.357749:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177201.357751:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177201.357752:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177201.357754:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177201.357758:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177201.357760:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177201.357764:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177201.357767:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177201.357770:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177201.357773:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235587968, portal 10 00000100:00000001:1.0:1786177201.357775:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177201.357776:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469630848 : -110371239920768 : ffff9b9e31737b80) 00000100:00000040:1.0:1786177201.357783:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357789:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177201.357792:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177201.357794:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177201.357796:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235587968, offset 0 00000400:00000200:1.0:1786177201.357800:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177201.357804:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177201.357808:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177201.357811:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177201.357814:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177201.357816:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177201.357819:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177201.357822:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426660:426660:256:4294967295] 192.168.202.132@tcp LPNI seq info [426622:426622:8:4294967295] 00000400:00000200:1.0:1786177201.357826:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177201.357833:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177201.357838:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177201.357841:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177201.357846:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177201.357849:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786177201.357854:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177201.357857:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1786177201.357860:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177201.357862:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177201.357865:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177201.357869:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177201.357872:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177201.357874:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177201.357877:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177201.357880:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357888:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177201.357891:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357898:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.357900:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177201.357904:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177201.357907:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177201.357910:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177201.357912:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306548228 00000100:00000001:1.0:1786177201.357915:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306552064 00000100:00000001:1.0:1786177201.357917:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306548228 00000100:00000001:1.0:1786177201.357919:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306552065 00000100:00000001:1.0:1786177201.357922:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177201.357924:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.357926:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177201.357931:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177201/real 1786177201] req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177201.357940:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177201.357942:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177201.357943:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177201.357946:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177201.357948:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177201.357950:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177201.357954:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177201.357963:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357971:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177201.357972:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.357974:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177201.357975:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177201.357978:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357982:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177201.357985:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.357990:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177201.357991:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.357992:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177201.357993:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.357994:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177201.357996:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177201.357998:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177201.358000:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177201.358002:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177201.358005:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177201.358010:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235587968:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177201.358013:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177201.358015:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358019:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177201.358020:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177201.358021:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177201.358022:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177201.358024:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177201.358026:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588096:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177201.358028:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177201.358030:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177201.358031:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177201.358032:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177201.358035:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177201.358036:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177201.358039:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177201.358040:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177201.358043:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177201.358045:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235588096, portal 10 00000100:00000001:1.0:1786177201.358046:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177201.358047:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469616512 : -110371239935104 : ffff9b9e31734380) 00000100:00000040:1.0:1786177201.358051:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358060:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177201.358061:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177201.358063:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177201.358064:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235588096, offset 0 00000400:00000200:1.0:1786177201.358067:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177201.358070:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177201.358072:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177201.358074:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177201.358076:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177201.358078:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177201.358079:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177201.358082:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426661:426661:256:4294967295] 192.168.202.132@tcp LPNI seq info [426623:426623:8:4294967295] 00000400:00000200:1.0:1786177201.358084:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177201.358088:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177201.358091:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177201.358093:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177201.358096:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177201.358098:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786177201.358101:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177201.358102:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1786177201.358105:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177201.358106:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177201.358108:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177201.358111:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177201.358113:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177201.358114:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177201.358116:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177201.358118:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358123:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177201.358125:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358134:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.358135:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177201.358137:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177201.358139:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177201.358141:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177201.358143:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306548228 00000100:00000001:1.0:1786177201.358144:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306552064 00000100:00000001:1.0:1786177201.358146:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306548228 00000100:00000001:1.0:1786177201.358147:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306552065 00000100:00000001:1.0:1786177201.358149:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177201.358150:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.358151:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177201.358154:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177201/real 1786177201] req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177201.358158:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177201.358159:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177201.358160:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177201.358161:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177201.358162:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177201.358163:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177201.358165:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177201.358170:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358174:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177201.358175:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358176:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177201.358177:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177201.358179:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358184:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177201.358187:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358191:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177201.358192:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358193:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177201.358194:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358195:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177201.358196:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177201.358198:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177201.358200:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177201.358201:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177201.358203:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177201.358208:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588096:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177201.358210:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177201.358212:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358216:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177201.358217:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177201.358218:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177201.358219:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177201.358221:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177201.358222:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588224:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177201.358224:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177201.358226:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177201.358227:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177201.358228:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177201.358230:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177201.358232:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177201.358234:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177201.358235:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177201.358237:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177201.358239:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235588224, portal 4 00000100:00000001:1.0:1786177201.358240:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177201.358241:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469624576 : -110371239927040 : ffff9b9e31736300) 00000100:00000040:1.0:1786177201.358245:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358249:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177201.358250:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177201.358252:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177201.358253:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235588224, offset 0 00000400:00000200:1.0:1786177201.358255:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177201.358257:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177201.358259:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177201.358261:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177201.358263:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177201.358264:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177201.358266:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177201.358268:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426662:426662:256:4294967295] 192.168.202.132@tcp LPNI seq info [426624:426624:8:4294967295] 00000400:00000200:1.0:1786177201.358270:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177201.358274:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177201.358277:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177201.358279:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177201.358281:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177201.358282:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786177201.358285:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177201.358286:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1786177201.358289:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177201.358305:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177201.358308:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177201.358310:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177201.358313:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177201.358314:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177201.358315:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177201.358318:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358323:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177201.358325:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358330:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.358331:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177201.358333:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177201.358335:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177201.358337:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177201.358339:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306548228 00000100:00000001:1.0:1786177201.358340:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306552064 00000100:00000001:1.0:1786177201.358342:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306548228 00000100:00000001:1.0:1786177201.358343:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306552065 00000100:00000001:1.0:1786177201.358345:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177201.358346:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.358347:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177201.358349:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177201/real 1786177201] req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177201.358354:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177201.358355:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177201.358356:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177201.358373:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177201.358374:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177201.358376:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177201.358378:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177201.358384:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358388:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177201.358389:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358390:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177201.358391:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177201.358393:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358397:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177201.358400:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358404:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177201.358406:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358407:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177201.358408:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358409:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177201.358410:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177201.358411:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177201.358413:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177201.358414:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177201.358416:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177201.358421:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588224:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177201.358423:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177201.358426:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358430:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177201.358431:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177201.358433:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177201.358434:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177201.358435:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177201.358437:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588352:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177201.358439:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177201.358440:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177201.358441:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177201.358442:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177201.358444:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177201.358446:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177201.358448:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177201.358449:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177201.358452:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177201.358453:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235588352, portal 4 00000100:00000001:1.0:1786177201.358455:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177201.358456:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469615616 : -110371239936000 : ffff9b9e31734000) 00000100:00000040:1.0:1786177201.358459:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358464:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177201.358465:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177201.358467:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177201.358468:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235588352, offset 0 00000400:00000200:1.0:1786177201.358470:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177201.358473:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177201.358475:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177201.358476:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177201.358478:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177201.358480:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177201.358481:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177201.358484:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426663:426663:256:4294967295] 192.168.202.132@tcp LPNI seq info [426625:426625:8:4294967295] 00000400:00000200:1.0:1786177201.358486:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177201.358490:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177201.358493:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177201.358496:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177201.358498:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177201.358500:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1786177201.358503:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177201.358505:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1786177201.358507:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177201.358509:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177201.358510:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177201.358513:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177201.358514:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177201.358516:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177201.358517:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177201.358519:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358524:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177201.358526:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177226 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358530:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.358531:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177201.358533:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177201.358534:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177201.358536:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177201.358537:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306548231 00000100:00000001:1.0:1786177201.358539:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306552064 00000100:00000001:1.0:1786177201.358540:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306548231 00000100:00000001:1.0:1786177201.358542:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306552065 00000100:00000001:1.0:1786177201.358543:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177201.358545:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.358546:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177201.358548:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177201/real 1786177201] req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177201.358553:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177201.358554:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177201.358555:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177201.358556:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177201.358557:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177201.358559:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177201.358562:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177201.358571:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358579:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177201.358580:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358582:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177201.358583:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177201.358587:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358594:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177201.358599:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358606:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177201.358609:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358610:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177201.358612:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358614:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177201.358615:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177201.358618:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177201.358622:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177201.358624:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177201.358627:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177201.358633:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588352:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177201.358635:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358636:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177201.358638:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737b80 x1872930235587968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358643:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177201.358644:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177201.358645:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177201.358647:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177201.358648:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177201.358649:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177201.358651:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177201.358652:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:1.0:1786177201.358654:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177201.358655:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177201.358656:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737b80. 00000100:00000001:1.0:1786177201.358658:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177201.358659:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358660:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177201.358662:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734380 x1872930235588096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358666:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177201.358667:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177201.358667:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177201.358669:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177201.358669:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177201.358670:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177201.358671:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177201.358673:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:1.0:1786177201.358674:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177201.358675:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177201.358676:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734380. 00000100:00000001:1.0:1786177201.358677:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177201.358678:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358679:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177201.358681:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736300 x1872930235588224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358685:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177201.358685:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177201.358686:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177201.358687:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177201.358688:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177201.358689:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177201.358690:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177201.358690:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:1.0:1786177201.358692:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177201.358692:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177201.358693:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736300. 00000100:00000001:1.0:1786177201.358694:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177201.358695:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358696:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177201.358698:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734000 x1872930235588352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177226 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177201.358701:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177201.358702:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177201.358703:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177201.358704:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177201.358705:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177201.358705:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177201.358706:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177201.358707:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:1.0:1786177201.358708:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177201.358709:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177201.358710:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734000. 00000100:00000001:1.0:1786177201.358712:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177201.358712:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358714:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.358716:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177201.358717:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.358719:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177201.358720:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177201.868460:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177201.868468:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177201.868471:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177201.868477:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177201.868481:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177201.868485:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.868488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177201.868491:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177201.868494:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177202.252474:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177202.252480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177202.252481:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177202.252484:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177202.252486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177202.252492:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177202.252493:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177202.252496:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177202.252497:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177202.252500:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177202.252504:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177202.252504:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177202.252509:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177202.252509:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177202.252516:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177202.252576:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177202.252580:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177202.252582:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177202.892469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177202.892477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177202.892479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177202.892485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177202.892489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177202.892492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177202.892495:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177202.892498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177202.892501:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177203.276478:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177203.276483:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177203.276487:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177203.276490:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177203.276491:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177203.276495:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1786177203.276500:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177203.276501:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177203.276507:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177203.276508:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177203.276511:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177203.276514:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177203.276515:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177203.276520:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177203.276521:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177203.276526:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177203.276528:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177203.276532:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177203.276532:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177203.276541:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177203.276543:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177203.276545:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177203.276547:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177203.276549:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177203.276550:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177203.276552:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177203.276555:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786177203.276598:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786177203.276601:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786177203.276604:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177203.916462:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177203.916470:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177203.916472:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177203.916476:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177203.916479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177203.916483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177203.916485:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177203.916488:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177203.916490:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177204.300459:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177204.300460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177204.300465:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177204.300469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177204.300470:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177204.300473:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177204.300475:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177204.300476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177204.300480:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177204.300485:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177204.300489:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177204.300492:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177204.300494:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177204.300496:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177204.300499:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177204.300535:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177204.300539:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177204.300542:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177204.940456:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177204.940463:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177204.940465:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177204.940470:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177204.940473:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177204.940476:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177204.940478:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177204.940480:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177204.940483:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177205.260443:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306552064, at 4306552128 00000100:00000040:1.0:1786177205.260443:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306552065, at 4306552128 00000100:00000010:0.0:1786177205.260459:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62e40. 00000100:00000010:1.0:1786177205.260462:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62960. 00000100:00000001:1.0:1786177205.324475:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177205.324482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177205.324483:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177205.324487:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177205.324488:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177205.324496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177205.324496:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177205.324499:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177205.324503:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177205.324506:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177205.324509:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177205.324509:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177205.324514:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177205.324519:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177205.324522:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786177205.324540:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1786177205.324544:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1786177205.324548:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177205.964463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177205.964471:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177205.964474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177205.964482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177205.964487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177205.964492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177205.964495:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177205.964498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177205.964503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.348504:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177206.348508:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177206.348514:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.348516:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177206.348521:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177206.348525:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177206.348528:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177206.348529:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177206.348535:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177206.348536:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177206.348539:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.348542:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.348548:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177206.348551:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.348555:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177206.348612:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177206.348617:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177206.348620:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177206.476540:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177206.476556:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177206.476560:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177206.476567:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177206.476572:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177206.476580:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11880 00000400:00000200:3.0:1786177206.476590:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177206.476597:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177206.476600:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177206.476602:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.476606:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177206.476609:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.476610:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177206.476614:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177206.476616:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177206.476619:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177206.476622:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177206.476624:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177206.476626:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177206.476628:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.476630:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177206.476631:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.476632:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177206.476635:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177206.476637:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177206.476638:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177206.476641:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177206.476654:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177206.476657:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.476659:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177206.476661:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177206.476673:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786177206.476676:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177206.476681:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177206.476683:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.476686:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177206.476689:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177206.476693:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177206.476705:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786177206.476713:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177206.476723:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177206.476733:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177206.476737:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177206.476740:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177206.476761:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177206.476763:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177206.476766:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177206.476767:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177206.476769:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177206.476771:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177206.476772:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11880 00000400:00000200:3.0:1786177206.476775:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177206.476777:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177206.476779:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177206.476781:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.476782:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177206.476784:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.476785:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177206.476787:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177206.476788:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177206.476789:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177206.476791:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177206.476794:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177206.476796:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177206.476799:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.476802:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177206.476803:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.476805:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177206.476806:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177206.476808:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177206.476809:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177206.476811:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177206.476813:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1786177206.476816:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177206.476820:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1786177206.476820:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.476822:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177206.476824:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1786177206.476828:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177206.476833:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177206.476835:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177206.476836:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177206.476841:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786177206.476845:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177206.476850:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177206.476852:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1786177206.476853:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.476856:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.476859:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177206.476862:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1786177206.476863:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177206.476866:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177206.476869:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177206.476869:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177206.476877:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235588480:192.168.202.132@tcp:250:kworker.0 02000000:00000010:3.0:1786177206.476878:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:1.0:1786177206.476880:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1786177206.476886:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177206.476887:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177206.476890:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177206.476894:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1786177206.476898:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1786177206.476900:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177206.476905:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.476913:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177206.476914:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177206.476918:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177206.476920:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000100:00000040:3.0:1786177206.476924:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177206.476938:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177206.476939:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00080000:3.0:1786177206.476941:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177206.476946:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177206.476947:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177206.476949:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1786177206.476950:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235588480, portal 25 00000100:00000001:3.0:1786177206.476951:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177206.476956:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1786177206.476957:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11880 00000400:00000200:3.0:1786177206.476961:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177206.476963:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177206.476967:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280821632 : -110371428729984 : ffff9b9e26327b80) 00000100:00080000:3.0:1786177206.476968:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177206.476973:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.476976:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177206.476982:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177206.476984:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177206.476984:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177206.476988:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177206.476990:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177206.476991:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177206.476993:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177206.476994:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177206.476998:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177206.477003:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.477008:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177206.477012:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.477015:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177206.477016:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177206.477023:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177206.477026:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000010:1.0:1786177206.477028:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00080000:3.0:1786177206.477033:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000200:1.0:1786177206.477036:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235588480, offset 0 00000100:00000001:3.0:1786177206.477036:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177206.477039:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177206.477042:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177206.477043:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.477045:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177206.477048:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177206.477051:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.477052:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177206.477057:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000400:00000200:1.0:1786177206.477059:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177206.477060:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177206.477065:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177206.477066:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.477068:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177206.477069:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177206.477070:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177206.477074:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000400:00000200:1.0:1786177206.477075:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177206.477078:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177206.477084:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177206.477087:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:3.0:1786177206.477100:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177206.477102:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1786177206.477103:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177206.477110:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177206.477114:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177206.477123:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1786177206.477130:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177206.477131:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00080000:3.0:1786177206.477135:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177206.477141:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:1.0:1786177206.477142:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426664:426664:256:4294967295] 192.168.202.132@tcp LPNI seq info [426626:426626:8:4294967295] 00000100:00000001:3.0:1786177206.477144:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177206.477146:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177206.477151:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177206.477156:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11880 00000400:00000200:3.0:1786177206.477159:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177206.477161:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177206.477165:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:3.0:1786177206.477165:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177206.477173:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.477175:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177206.477180:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.477186:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177206.477189:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000020:00000001:3.0:1786177206.477191:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177206.477196:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177206.477198:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:1.0:1786177206.477203:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177206.477204:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177206.477209:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177206.477212:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000800:00000010:1.0:1786177206.477213:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000040:3.0:1786177206.477218:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.477222:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177206.477227:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:1.0:1786177206.477229:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177206.477230:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177206.477236:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000800:00000200:1.0:1786177206.477238:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00080000:3.0:1786177206.477239:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177206.477258:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177206.477260:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177206.477265:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.1:1786177206.477265:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786177206.477270:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.477274:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:1.1:1786177206.477274:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000008:00100000:3.0:1786177206.477276:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177206.477282:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.477284:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1786177206.477289:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000010:3.0:1786177206.477289:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786177206.477294:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000800:00000010:1.0:1786177206.477296:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 02000000:00000001:3.0:1786177206.477297:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177206.477300:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.477303:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1786177206.477308:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:3.0:1786177206.477309:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177206.477314:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177206.477318:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000400:00000200:1.0:1786177206.477319:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1786177206.477325:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177206.477329:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177206.477335:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00100000:3.0:1786177206.477336:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177206.477344:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786177206.477348:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1786177206.477354:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177206.477354:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177206.477394:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177206.477401:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177206.477406:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177206.477413:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177206.477416:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177206.477420:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177206.477423:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1786177206.477424:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177206.477428:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11880 00000400:00000200:3.0:1786177206.477433:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177206.477434:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1786177206.477438:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177206.477443:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177206.477449:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.477454:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177206.477459:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177206.477461:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.477461:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177206.477467:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1786177206.477467:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177206.477471:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177206.477473:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177206.477475:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177206.477477:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177206.477478:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177206.477480:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177206.477481:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177206.477486:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177206.477491:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177206.477493:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177206.477494:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177206.477499:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:1.0:1786177206.477500:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306557185 00000100:00080000:3.0:1786177206.477502:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177206.477507:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177206.477508:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:1.0:1786177206.477511:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62960. 02000000:00000001:3.0:1786177206.477511:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177206.477516:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.477521:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177206.477523:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177206.477524:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000008:00000001:3.0:1786177206.477529:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.477533:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.477537:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306557185 00000100:00000010:3.0:1786177206.477539:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000100:00000010:1.0:1786177206.477546:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62000. 00000020:00000040:3.0:1786177206.477546:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177206.477554:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177206.477555:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1786177206.477556:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177206.477561:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177206.477563:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177206.477564:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177206.477569:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.477570:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.477575:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000010:3.0:1786177206.477576:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786177206.477581:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177206.477583:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177206/real 1786177206] req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786177206.477588:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177206.477616:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177206.477618:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1786177206.477619:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1786177206.477621:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177206.477623:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177206.477629:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000100:00000001:3.0:1786177206.477631:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177206.477634:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000100:00000040:3.0:1786177206.477637:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11891) 00000400:00000010:1.0:1786177206.477639:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177206.477645:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177206.477652:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177206.477669:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.477678:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177206.477680:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.477683:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177206.477686:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177206.477690:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.477698:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177206.477704:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.477712:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177206.477715:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.477717:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177206.477719:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.477722:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177206.477724:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177206.477728:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177206.477732:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177206.477735:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177206.477740:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177206.477750:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235588480:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177206.477756:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.477759:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177206.477763:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327b80 x1872930235588480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.477777:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177206.477780:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177206.477783:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177206.477787:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177206.477790:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177206.477792:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177206.477794:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177206.477797:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:1.0:1786177206.477802:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177206.477804:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177206.477806:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327b80. 00000100:00000001:1.0:1786177206.477810:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177206.477811:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.477814:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.477822:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177206.477825:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.477827:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177206.477829:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177206.477830:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177206.477835:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.477842:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177206.477843:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177206.477846:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177206.477848:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177206.477850:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177206.477854:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588608:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177206.477857:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177206.477859:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177206.477861:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177206.477863:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177206.477867:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177206.477869:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177206.477873:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177206.477877:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177206.477880:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177206.477886:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235588608, portal 10 00000100:00000001:1.0:1786177206.477888:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177206.477890:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280812672 : -110371428738944 : ffff9b9e26325880) 00000100:00000040:1.0:1786177206.477897:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.477905:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177206.477907:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177206.477910:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177206.477912:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235588608, offset 0 00000400:00000200:1.0:1786177206.477917:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177206.477921:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177206.477925:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177206.477929:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177206.477931:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177206.477934:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177206.477937:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177206.477940:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426665:426665:256:4294967295] 192.168.202.132@tcp LPNI seq info [426627:426627:8:4294967295] 00000400:00000200:1.0:1786177206.477945:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177206.477952:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177206.477957:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177206.477961:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177206.477966:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177206.477969:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177206.477974:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177206.477977:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177206.477981:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177206.477983:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177206.477987:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177206.477991:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177206.477994:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177206.477996:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177206.478000:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177206.478004:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478012:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177206.478015:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478023:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.478024:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177206.478028:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177206.478032:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177206.478036:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177206.478038:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306553348 00000100:00000001:1.0:1786177206.478041:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306557185 00000100:00000001:1.0:1786177206.478044:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306553348 00000100:00000001:1.0:1786177206.478046:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306557185 00000100:00000001:1.0:1786177206.478049:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177206.478051:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.478053:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177206.478057:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177206/real 1786177206] req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177206.478065:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177206.478067:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177206.478069:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177206.478071:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177206.478073:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177206.478076:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177206.478079:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177206.478088:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478102:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177206.478103:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.478106:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177206.478107:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177206.478111:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478117:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177206.478121:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478129:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177206.478131:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.478133:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177206.478134:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.478136:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177206.478139:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177206.478142:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177206.478145:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177206.478148:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177206.478152:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177206.478160:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588608:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177206.478165:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177206.478169:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478176:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177206.478178:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177206.478181:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177206.478183:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177206.478185:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177206.478188:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588736:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177206.478192:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177206.478194:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177206.478195:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177206.478197:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177206.478202:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177206.478204:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177206.478208:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177206.478210:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177206.478214:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177206.478217:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235588736, portal 10 00000100:00000001:1.0:1786177206.478220:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177206.478221:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280811776 : -110371428739840 : ffff9b9e26325500) 00000100:00000040:1.0:1786177206.478227:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478234:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177206.478237:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177206.478239:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177206.478256:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235588736, offset 0 00000400:00000200:1.0:1786177206.478262:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177206.478267:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177206.478271:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177206.478274:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177206.478277:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177206.478279:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177206.478282:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177206.478285:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426666:426666:256:4294967295] 192.168.202.132@tcp LPNI seq info [426628:426628:8:4294967295] 00000400:00000200:1.0:1786177206.478290:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177206.478297:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177206.478302:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177206.478306:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177206.478311:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177206.478314:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177206.478319:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177206.478322:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177206.478326:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177206.478328:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177206.478332:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177206.478335:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177206.478339:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177206.478341:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177206.478344:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177206.478348:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478376:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177206.478380:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478388:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.478390:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177206.478394:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177206.478398:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177206.478402:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177206.478405:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306553349 00000100:00000001:1.0:1786177206.478408:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306557185 00000100:00000001:1.0:1786177206.478410:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306553349 00000100:00000001:1.0:1786177206.478413:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306557185 00000100:00000001:1.0:1786177206.478415:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177206.478417:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.478419:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177206.478424:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177206/real 1786177206] req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177206.478433:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177206.478435:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177206.478437:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177206.478439:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177206.478441:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177206.478443:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177206.478447:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177206.478457:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478464:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177206.478465:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.478467:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177206.478468:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177206.478472:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478480:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177206.478484:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478491:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177206.478494:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.478496:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177206.478497:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.478499:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177206.478501:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177206.478505:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177206.478508:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177206.478511:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177206.478517:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177206.478526:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588736:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177206.478531:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177206.478535:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478542:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177206.478544:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177206.478546:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177206.478548:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177206.478551:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177206.478554:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588864:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177206.478558:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177206.478560:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177206.478562:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177206.478564:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177206.478568:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177206.478571:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177206.478574:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177206.478577:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177206.478581:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177206.478584:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235588864, portal 4 00000100:00000001:1.0:1786177206.478586:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177206.478588:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280806400 : -110371428745216 : ffff9b9e26324000) 00000100:00000040:1.0:1786177206.478594:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478601:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177206.478604:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177206.478606:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177206.478609:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235588864, offset 0 00000400:00000200:1.0:1786177206.478614:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177206.478618:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177206.478622:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177206.478625:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177206.478628:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177206.478631:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177206.478633:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177206.478637:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426667:426667:256:4294967295] 192.168.202.132@tcp LPNI seq info [426629:426629:8:4294967295] 00000400:00000200:1.0:1786177206.478642:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177206.478648:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177206.478653:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177206.478656:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177206.478661:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177206.478664:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177206.478669:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177206.478672:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177206.478676:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177206.478679:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177206.478682:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177206.478686:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177206.478690:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177206.478692:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177206.478695:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177206.478699:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478707:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177206.478710:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478716:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.478718:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177206.478721:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177206.478725:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177206.478728:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177206.478731:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306553349 00000100:00000001:1.0:1786177206.478734:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306557185 00000100:00000001:1.0:1786177206.478736:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306553349 00000100:00000001:1.0:1786177206.478738:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306557185 00000100:00000001:1.0:1786177206.478741:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177206.478743:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.478745:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177206.478749:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177206/real 1786177206] req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177206.478756:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177206.478758:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177206.478760:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177206.478762:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177206.478764:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177206.478766:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177206.478770:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177206.478779:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478786:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177206.478787:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.478789:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177206.478791:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177206.478794:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478801:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177206.478805:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478813:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177206.478817:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.478819:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177206.478820:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.478822:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177206.478824:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177206.478827:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177206.478831:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177206.478834:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177206.478838:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177206.478848:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588864:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177206.478852:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177206.478856:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478864:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177206.478866:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177206.478868:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177206.478870:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177206.478873:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177206.478876:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588992:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177206.478880:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177206.478882:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177206.478883:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177206.478886:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177206.478889:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177206.478892:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177206.478895:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177206.478911:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177206.478916:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177206.478919:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235588992, portal 4 00000100:00000001:1.0:1786177206.478926:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177206.478928:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809088 : -110371428742528 : ffff9b9e26324a80) 00000100:00000040:1.0:1786177206.478934:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.478942:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177206.478945:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177206.478947:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177206.478950:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235588992, offset 0 00000400:00000200:1.0:1786177206.478954:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177206.478958:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177206.478962:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177206.478965:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177206.478968:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177206.478971:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177206.478973:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177206.478977:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426668:426668:256:4294967295] 192.168.202.132@tcp LPNI seq info [426630:426630:8:4294967295] 00000400:00000200:1.0:1786177206.478980:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177206.478985:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177206.478988:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177206.478991:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177206.478994:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177206.478996:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177206.478999:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177206.479001:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177206.479003:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177206.479005:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177206.479007:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177206.479010:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177206.479012:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177206.479013:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177206.479015:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177206.479017:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.479027:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177206.479029:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177231 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.479036:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.479038:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177206.479041:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177206.479044:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177206.479047:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177206.479050:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306553351 00000100:00000001:1.0:1786177206.479052:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306557185 00000100:00000001:1.0:1786177206.479054:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306553351 00000100:00000001:1.0:1786177206.479056:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306557185 00000100:00000001:1.0:1786177206.479059:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177206.479061:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.479063:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177206.479067:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177206/real 1786177206] req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177206.479075:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177206.479076:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177206.479078:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177206.479081:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177206.479083:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177206.479086:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177206.479089:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177206.479098:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.479105:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177206.479107:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.479109:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177206.479111:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177206.479115:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.479123:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177206.479127:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.479133:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177206.479135:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.479136:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177206.479138:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.479139:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177206.479140:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177206.479141:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177206.479144:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177206.479146:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177206.479148:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177206.479153:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235588992:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177206.479156:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.479157:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177206.479159:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325880 x1872930235588608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.479164:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177206.479165:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177206.479167:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177206.479169:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177206.479170:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177206.479172:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177206.479174:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177206.479175:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:1.0:1786177206.479177:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177206.479178:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177206.479179:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325880. 00000100:00000001:1.0:1786177206.479181:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177206.479182:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.479184:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177206.479186:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325500 x1872930235588736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.479190:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177206.479191:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177206.479192:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177206.479194:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177206.479194:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177206.479195:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177206.479196:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177206.479197:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:1.0:1786177206.479198:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177206.479199:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177206.479200:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325500. 00000100:00000001:1.0:1786177206.479201:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177206.479202:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.479203:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177206.479205:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324000 x1872930235588864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.479208:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177206.479209:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177206.479210:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177206.479211:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177206.479212:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177206.479212:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177206.479213:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177206.479214:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 02000000:00000001:1.0:1786177206.479216:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177206.479217:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177206.479218:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324000. 00000100:00000001:1.0:1786177206.479220:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177206.479221:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.479222:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177206.479224:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324a80 x1872930235588992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177231 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177206.479227:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177206.479228:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177206.479229:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177206.479231:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177206.479231:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177206.479232:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177206.479233:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177206.479234:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:1.0:1786177206.479235:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177206.479235:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177206.479236:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324a80. 00000100:00000001:1.0:1786177206.479237:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177206.479238:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.479239:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.479255:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177206.479257:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.479260:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177206.479261:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177206.988485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177206.988497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177206.988500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177206.988508:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177206.988513:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177206.988518:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.988522:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177206.988525:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177206.988529:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177207.372503:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177207.372504:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177207.372513:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177207.372515:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177207.372519:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177207.372522:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177207.372525:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177207.372527:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177207.372529:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177207.372532:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177207.372535:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177207.372541:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177207.372545:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177207.372549:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177207.372553:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177207.372597:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177207.372602:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177207.372606:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177208.012456:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177208.012466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177208.012468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177208.012474:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177208.012478:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177208.012482:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177208.012485:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177208.012488:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177208.012491:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177208.396499:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177208.396502:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177208.396509:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177208.396511:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177208.396515:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177208.396516:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177208.396518:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177208.396524:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177208.396526:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177208.396526:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177208.396533:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177208.396538:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177208.396542:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177208.396546:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177208.396550:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177208.396601:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177208.396607:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177208.396612:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177209.036492:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177209.036503:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177209.036506:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177209.036514:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177209.036520:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177209.036524:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177209.036527:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177209.036530:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177209.036534:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177209.420408:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177209.420416:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177209.420419:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177209.420425:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177209.420429:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177209.420432:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177209.420435:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177209.420438:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177209.420441:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177209.420460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177209.420466:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177209.420471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177209.420472:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177209.420476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177209.420478:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177209.420523:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177209.420527:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177209.420530:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177210.060493:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177210.060504:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177210.060507:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177210.060525:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177210.060531:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177210.060536:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177210.060540:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177210.060544:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177210.060548:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177210.380485:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306557185, at 4306557248 00000100:00000040:1.0:1786177210.380485:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306557185, at 4306557248 00000100:00000010:0.0:1786177210.380503:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62960. 00000100:00000010:1.0:1786177210.380505:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62000. 00000100:00000001:1.0:1786177210.444491:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177210.444496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177210.444500:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177210.444500:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177210.444504:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177210.444508:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177210.444509:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177210.444514:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177210.444515:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177210.444519:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177210.444520:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177210.444523:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177210.444524:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177210.444527:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177210.444529:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177210.444530:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177210.444535:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177210.444538:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177210.444538:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177210.444542:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177210.444544:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177210.444624:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177210.444628:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177210.444631:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177211.084468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177211.084480:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177211.084484:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177211.084492:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177211.084497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177211.084502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177211.084505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177211.084509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177211.084512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.468490:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177211.468494:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177211.468499:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.468502:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177211.468505:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177211.468507:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177211.468508:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177211.468515:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177211.468516:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177211.468518:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177211.468524:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177211.468529:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.468533:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177211.468537:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.468541:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177211.468592:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177211.468596:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177211.468600:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177211.596474:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177211.596486:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177211.596489:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177211.596494:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177211.596499:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177211.596506:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11886 00000400:00000200:3.0:1786177211.596516:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177211.596524:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177211.596528:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177211.596531:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596535:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177211.596537:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596540:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177211.596545:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177211.596548:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177211.596551:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177211.596554:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177211.596557:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177211.596560:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177211.596563:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596565:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177211.596568:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596570:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177211.596573:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177211.596577:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596579:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177211.596583:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177211.596587:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177211.596590:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177211.596592:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177211.596595:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177211.596605:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786177211.596609:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177211.596614:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177211.596616:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177211.596620:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177211.596622:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177211.596627:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177211.596636:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786177211.596643:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177211.596653:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177211.596664:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177211.596668:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177211.596673:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177211.596698:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177211.596702:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177211.596705:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177211.596707:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177211.596709:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177211.596712:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177211.596714:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11886 00000400:00000200:3.0:1786177211.596718:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177211.596722:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177211.596725:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177211.596729:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596732:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177211.596735:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596738:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177211.596741:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177211.596742:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000100:00000001:1.0:1786177211.596744:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1786177211.596745:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177211.596748:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00000001:1.0:1786177211.596750:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1786177211.596752:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177211.596755:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1786177211.596756:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177211.596762:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596765:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177211.596766:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177211.596769:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596775:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177211.596779:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177211.596782:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596785:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177211.596788:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177211.596798:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177211.596802:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177211.596804:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177211.596807:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1786177211.596809:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177211.596812:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1786177211.596813:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1786177211.596818:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1786177211.596818:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177211.596824:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177211.596825:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235589120:192.168.202.132@tcp:250:kworker.0 00000100:00000001:3.0:1786177211.596827:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.596834:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:3.0:1786177211.596834:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786177211.596839:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177211.596840:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1786177211.596843:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177211.596844:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177211.596846:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177211.596849:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1786177211.596850:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177211.596853:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:1.0:1786177211.596856:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:3.0:1786177211.596856:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177211.596861:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177211.596861:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786177211.596867:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177211.596869:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:3.0:1786177211.596875:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177211.596876:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177211.596885:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000001:3.0:1786177211.596887:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177211.596891:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177211.596894:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235589120, portal 25 00000100:00000040:3.0:1786177211.596897:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.596899:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177211.596903:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280808192 : -110371428743424 : ffff9b9e26324700) 00000100:00000040:1.0:1786177211.596916:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177211.596925:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177211.596930:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177211.596934:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1786177211.596935:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177211.596939:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:1.0:1786177211.596943:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177211.596943:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177211.596949:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:1.0:1786177211.596950:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00080000:3.0:1786177211.596955:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11886 00000100:00000200:1.0:1786177211.596956:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235589120, offset 0 00000400:00000200:3.0:1786177211.596962:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177211.596969:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:1.0:1786177211.596970:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177211.596972:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177211.596976:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596978:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177211.596981:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177211.596983:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.596986:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177211.596988:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1786177211.596991:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000040:3.0:1786177211.596991:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177211.596995:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177211.596998:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:1.0:1786177211.596999:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1786177211.597002:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177211.597005:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000001:3.0:1786177211.597007:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:1.0:1786177211.597011:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1786177211.597014:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177211.597017:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.597018:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177211.597024:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177211.597026:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426669:426669:256:4294967295] 192.168.202.132@tcp LPNI seq info [426631:426631:8:4294967295] 00000100:00000001:3.0:1786177211.597027:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177211.597032:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:1.0:1786177211.597036:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:3.0:1786177211.597040:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177211.597043:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177211.597049:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1786177211.597052:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 02000000:00000001:3.0:1786177211.597053:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177211.597059:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177211.597063:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177211.597064:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177211.597068:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000800:00000010:1.0:1786177211.597069:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000008:00000001:3.0:1786177211.597071:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177211.597074:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177211.597078:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000010:3.0:1786177211.597079:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000800:00000200:1.0:1786177211.597082:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000020:00000040:3.0:1786177211.597084:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177211.597090:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.1:1786177211.597090:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786177211.597093:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786177211.597095:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1786177211.597097:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:1.0:1786177211.597102:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 02000000:00000001:3.0:1786177211.597102:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177211.597106:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177211.597107:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177211.597113:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177211.597116:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177211.597120:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177211.597123:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 02000000:00000010:3.0:1786177211.597123:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:1.0:1786177211.597128:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786177211.597130:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177211.597134:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786177211.597140:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177211.597146:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177211.597153:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177211.597155:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177211.597159:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597164:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597167:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1786177211.597169:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.597172:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177211.597179:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177211.597183:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306562304 00000100:00000001:3.0:1786177211.597187:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177211.597190:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62000. 00000100:00080000:3.0:1786177211.597204:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177211.597210:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177211.597212:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177211.597214:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177211.597215:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1786177211.597219:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177211.597223:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11886 00000100:00000001:1.0:1786177211.597224:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306562304 00000100:00000010:1.0:1786177211.597229:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62600. 00000400:00000200:3.0:1786177211.597231:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177211.597233:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177211.597237:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1786177211.597237:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177211.597242:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177211.597243:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597248:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1786177211.597250:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.597255:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1786177211.597256:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177211/real 1786177211] req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177211.597260:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.597266:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177211.597270:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177211.597273:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177211.597276:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177211.597280:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000400:00000001:1.0:1786177211.597281:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1786177211.597283:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000001:1.0:1786177211.597284:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177211.597287:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000020:00000001:3.0:1786177211.597288:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000200:1.0:1786177211.597291:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177211.597294:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000040:3.0:1786177211.597295:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177211.597299:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1786177211.597299:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177211.597305:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:1.0:1786177211.597306:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177211.597308:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177211.597314:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177211.597321:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000200:1.0:1786177211.597322:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177211.597325:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177211.597330:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177211.597334:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1786177211.597335:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177211.597336:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.597341:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1786177211.597341:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597344:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177211.597346:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:1.0:1786177211.597349:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1786177211.597351:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177211.597378:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177211.597382:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1786177211.597388:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597390:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177211.597391:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597396:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1786177211.597398:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000100:00000001:1.0:1786177211.597399:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177211.597402:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1786177211.597403:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177211.597406:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1786177211.597407:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177211.597409:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1786177211.597410:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177211.597414:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:1.0:1786177211.597415:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1786177211.597417:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177211.597420:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177211.597421:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 02000000:00000010:3.0:1786177211.597424:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:1.0:1786177211.597425:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1786177211.597431:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177211.597432:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786177211.597439:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1786177211.597447:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235589120:192.168.202.132@tcp:250:kworker.0 00000100:00000001:3.0:1786177211.597450:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177211.597453:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597454:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177211.597458:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.597459:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177211.597465:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324700 x1872930235589120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177211.597471:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177211.597474:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1786177211.597476:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1786177211.597478:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:1.0:1786177211.597480:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1786177211.597480:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177211.597484:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:1.0:1786177211.597486:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 00000100:00000001:3.0:1786177211.597487:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177211.597491:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11886 02000000:00000001:1.0:1786177211.597493:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:3.0:1786177211.597497:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:1.0:1786177211.597498:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:3.0:1786177211.597501:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000020:00000040:1.0:1786177211.597503:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00080000:3.0:1786177211.597504:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177211.597509:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000020:00000001:1.0:1786177211.597510:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:3.0:1786177211.597511:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177211.597515:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000010:1.0:1786177211.597516:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786177211.597516:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177211.597522:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177211.597523:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 02000000:00000001:1.0:1786177211.597524:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1786177211.597525:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177211.597528:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 02000000:00000001:1.0:1786177211.597529:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:3.0:1786177211.597531:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000010:1.0:1786177211.597533:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324700. 00000020:00000001:3.0:1786177211.597534:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177211.597540:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177211.597541:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1786177211.597542:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177211.597544:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177211.597545:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177211.597550:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177211.597551:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1786177211.597552:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177211.597558:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177211.597559:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177211.597560:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177211.597562:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1786177211.597563:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177211.597567:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597568:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177211.597569:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177211.597571:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177211.597574:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597575:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1786177211.597576:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597580:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:3.0:1786177211.597580:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000100:00000001:1.0:1786177211.597584:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1786177211.597584:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177211.597588:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177211.597590:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177211.597593:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:1.0:1786177211.597594:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177211.597594:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177211.597597:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177211.597601:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786177211.597605:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597611:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177211.597613:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177211.597615:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177211.597618:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177211.597622:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177211.597626:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1786177211.597628:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235589248:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177211.597638:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177211.597642:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1786177211.597644:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1786177211.597645:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177211.597649:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177211.597650:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177211.597656:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 00000100:00000040:3.0:1786177211.597658:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177211.597660:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177211.597669:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177211.597675:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000100:00000001:3.0:1786177211.597677:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177211.597682:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000040:3.0:1786177211.597683:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11896) 00000100:00000200:1.0:1786177211.597690:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235589248, portal 10 00000100:00000001:1.0:1786177211.597693:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177211.597695:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280817152 : -110371428734464 : ffff9b9e26326a00) 00000100:00000040:1.0:1786177211.597703:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.597710:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177211.597713:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177211.597716:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177211.597719:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235589248, offset 0 00000400:00000200:1.0:1786177211.597723:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177211.597727:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177211.597731:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177211.597734:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177211.597737:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177211.597740:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177211.597742:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177211.597745:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426670:426670:256:4294967295] 192.168.202.132@tcp LPNI seq info [426632:426632:8:4294967295] 00000400:00000200:1.0:1786177211.597750:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177211.597756:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177211.597761:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177211.597765:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177211.597769:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177211.597773:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786177211.597778:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177211.597781:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1786177211.597784:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177211.597786:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177211.597790:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177211.597794:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177211.597798:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177211.597800:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177211.597803:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177211.597806:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.597815:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177211.597819:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.597826:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597827:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177211.597831:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177211.597835:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177211.597839:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177211.597841:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306558468 00000100:00000001:1.0:1786177211.597844:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306562304 00000100:00000001:1.0:1786177211.597847:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306558468 00000100:00000001:1.0:1786177211.597849:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306562304 00000100:00000001:1.0:1786177211.597852:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177211.597854:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.597856:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177211.597861:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177211/real 1786177211] req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177211.597870:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177211.597871:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177211.597874:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177211.597877:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177211.597878:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177211.597881:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177211.597884:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177211.597893:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.597901:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177211.597902:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.597903:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177211.597905:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177211.597908:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.597915:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177211.597919:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.597926:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177211.597929:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.597931:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177211.597932:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.597934:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177211.597937:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177211.597940:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177211.597943:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177211.597945:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177211.597949:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177211.597956:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235589248:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177211.597960:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177211.597964:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.597971:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177211.597973:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177211.597975:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177211.597977:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177211.597979:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177211.597982:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235589376:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177211.597986:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177211.597988:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177211.597990:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177211.597992:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177211.597995:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177211.597997:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177211.598000:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177211.598003:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177211.598006:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177211.598009:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235589376, portal 10 00000100:00000001:1.0:1786177211.598011:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177211.598012:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809984 : -110371428741632 : ffff9b9e26324e00) 00000100:00000040:1.0:1786177211.598019:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598026:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177211.598029:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177211.598031:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177211.598033:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235589376, offset 0 00000400:00000200:1.0:1786177211.598038:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177211.598042:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177211.598045:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177211.598048:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177211.598051:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177211.598053:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177211.598055:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177211.598058:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426671:426671:256:4294967295] 192.168.202.132@tcp LPNI seq info [426633:426633:8:4294967295] 00000400:00000200:1.0:1786177211.598063:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177211.598069:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177211.598074:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177211.598077:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177211.598081:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177211.598084:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786177211.598088:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177211.598091:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1786177211.598095:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177211.598097:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177211.598101:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177211.598104:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177211.598107:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177211.598109:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177211.598111:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177211.598115:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598124:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177211.598127:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598133:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.598135:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177211.598139:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177211.598142:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177211.598145:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177211.598148:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306558468 00000100:00000001:1.0:1786177211.598150:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306562304 00000100:00000001:1.0:1786177211.598153:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306558468 00000100:00000001:1.0:1786177211.598155:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306562304 00000100:00000001:1.0:1786177211.598158:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177211.598160:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.598162:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177211.598166:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177211/real 1786177211] req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177211.598175:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177211.598176:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177211.598178:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177211.598181:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177211.598183:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177211.598185:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177211.598189:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177211.598216:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598223:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177211.598225:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598227:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177211.598228:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177211.598232:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598240:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177211.598244:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598253:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177211.598255:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598257:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177211.598258:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598261:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177211.598263:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177211.598265:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177211.598268:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177211.598270:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177211.598274:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177211.598282:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235589376:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177211.598286:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177211.598290:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598298:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177211.598299:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177211.598302:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177211.598304:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177211.598306:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177211.598309:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235589504:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177211.598313:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177211.598315:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177211.598317:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177211.598319:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177211.598323:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177211.598325:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177211.598329:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177211.598332:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177211.598335:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177211.598338:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235589504, portal 4 00000100:00000001:1.0:1786177211.598340:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177211.598341:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809088 : -110371428742528 : ffff9b9e26324a80) 00000100:00000040:1.0:1786177211.598347:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598353:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177211.598374:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177211.598378:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177211.598381:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235589504, offset 0 00000400:00000200:1.0:1786177211.598386:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177211.598390:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177211.598394:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177211.598397:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177211.598400:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177211.598403:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177211.598406:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177211.598409:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426672:426672:256:4294967295] 192.168.202.132@tcp LPNI seq info [426634:426634:8:4294967295] 00000400:00000200:1.0:1786177211.598413:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177211.598420:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177211.598425:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177211.598429:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177211.598433:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177211.598436:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786177211.598439:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177211.598441:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1786177211.598443:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177211.598445:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177211.598448:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177211.598452:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177211.598455:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177211.598457:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177211.598460:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177211.598464:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598473:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177211.598475:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598489:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.598491:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177211.598494:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177211.598497:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177211.598501:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177211.598503:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306558469 00000100:00000001:1.0:1786177211.598506:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306562304 00000100:00000001:1.0:1786177211.598508:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306558469 00000100:00000001:1.0:1786177211.598510:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306562304 00000100:00000001:1.0:1786177211.598512:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177211.598515:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.598517:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177211.598521:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177211/real 1786177211] req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177211.598529:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177211.598530:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177211.598532:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177211.598535:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177211.598536:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177211.598538:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177211.598542:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177211.598551:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598558:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177211.598560:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598562:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177211.598563:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177211.598567:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598575:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177211.598579:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598586:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177211.598589:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598591:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177211.598592:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598594:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177211.598595:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177211.598598:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177211.598602:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177211.598605:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177211.598609:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177211.598617:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235589504:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177211.598622:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598624:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177211.598627:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326a00 x1872930235589248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598634:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177211.598636:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177211.598638:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177211.598641:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177211.598643:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177211.598644:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177211.598646:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177211.598648:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa800. 02000000:00000001:1.0:1786177211.598652:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177211.598653:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177211.598655:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326a00. 00000100:00000001:1.0:1786177211.598657:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177211.598659:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598660:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177211.598664:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324e00 x1872930235589376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598670:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177211.598672:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177211.598673:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177211.598676:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177211.598677:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177211.598678:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177211.598679:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177211.598680:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9000. 02000000:00000001:1.0:1786177211.598682:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177211.598682:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177211.598683:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324e00. 00000100:00000001:1.0:1786177211.598685:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177211.598686:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598687:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177211.598689:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324a80 x1872930235589504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598693:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177211.598694:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177211.598695:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177211.598696:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177211.598697:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177211.598697:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177211.598698:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177211.598699:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9400. 02000000:00000001:1.0:1786177211.598700:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177211.598701:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177211.598702:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324a80. 00000100:00000001:1.0:1786177211.598703:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177211.598704:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598706:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598708:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177211.598709:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.598711:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177211.598712:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177211.598713:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177211.598715:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598720:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177211.598721:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177211.598722:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177211.598723:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177211.598725:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177211.598726:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235589632:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177211.598728:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177211.598730:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177211.598731:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177211.598732:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177211.598734:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177211.598736:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177211.598738:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177211.598740:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177211.598742:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177211.598745:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235589632, portal 4 00000100:00000001:1.0:1786177211.598746:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177211.598747:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280806400 : -110371428745216 : ffff9b9e26324000) 00000100:00000040:1.0:1786177211.598751:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598755:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177211.598757:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177211.598758:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177211.598759:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235589632, offset 0 00000400:00000200:1.0:1786177211.598762:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177211.598765:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177211.598768:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177211.598770:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177211.598772:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177211.598774:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177211.598775:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177211.598777:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426673:426673:256:4294967295] 192.168.202.132@tcp LPNI seq info [426635:426635:8:4294967295] 00000400:00000200:1.0:1786177211.598781:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177211.598787:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177211.598792:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177211.598796:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177211.598801:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177211.598803:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1786177211.598808:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177211.598811:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1786177211.598815:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177211.598817:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177211.598821:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177211.598825:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177211.598828:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177211.598830:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177211.598833:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177211.598837:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598845:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177211.598848:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177236 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598855:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.598857:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177211.598860:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177211.598863:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177211.598867:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177211.598869:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306558471 00000100:00000001:1.0:1786177211.598872:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306562304 00000100:00000001:1.0:1786177211.598875:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306558471 00000100:00000001:1.0:1786177211.598877:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306562304 00000100:00000001:1.0:1786177211.598879:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177211.598882:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.598884:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177211.598888:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177211/real 1786177211] req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177211.598897:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177211.598898:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177211.598900:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177211.598903:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177211.598905:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177211.598907:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177211.598911:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177211.598920:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598927:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177211.598929:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598931:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177211.598933:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177211.598937:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598944:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177211.598948:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.598955:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177211.598959:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598960:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177211.598962:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598964:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177211.598965:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177211.598968:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177211.598971:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177211.598974:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177211.598978:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177211.598987:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235589632:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177211.598991:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.598993:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177211.598996:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324000 x1872930235589632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177236 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177211.599002:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177211.599003:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177211.599005:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177211.599008:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177211.599009:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177211.599010:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177211.599012:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177211.599014:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa000. 02000000:00000001:1.0:1786177211.599017:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177211.599018:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177211.599020:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324000. 00000100:00000001:1.0:1786177211.599022:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177211.599023:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.599025:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177211.599029:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177211.599030:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177211.599032:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177211.599034:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177212.108453:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177212.108462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177212.108465:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177212.108471:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177212.108475:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177212.108488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177212.108491:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177212.108494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177212.108498:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177212.492428:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177212.492434:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177212.492436:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177212.492441:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177212.492444:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177212.492447:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177212.492448:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177212.492450:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177212.492453:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177212.492454:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177212.492457:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177212.492458:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177212.492461:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177212.492465:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177212.492467:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177212.492469:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177212.492470:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177212.492474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177212.492475:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177212.492478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177212.492479:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177212.492480:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177212.492481:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177212.492483:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177212.492484:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177212.492485:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177212.492486:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177212.492524:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177212.492528:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177212.492531:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177213.132458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177213.132466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177213.132469:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177213.132475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177213.132480:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177213.132484:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177213.132488:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177213.132491:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177213.132494:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177213.516418:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177213.516424:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177213.516426:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177213.516431:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177213.516432:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177213.516435:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177213.516437:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177213.516439:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177213.516443:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177213.516443:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177213.516446:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177213.516450:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177213.516451:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177213.516454:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177213.516457:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177213.516503:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177213.516506:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177213.516509:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177214.156454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177214.156461:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177214.156463:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177214.156467:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177214.156471:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177214.156474:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177214.156477:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177214.156479:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177214.156481:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177214.540453:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177214.540458:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177214.540460:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177214.540463:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177214.540463:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177214.540470:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177214.540471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177214.540475:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177214.540476:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177214.540479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177214.540480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177214.540482:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177214.540486:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177214.540489:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177214.540491:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177214.540534:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177214.540538:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177214.540541:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177215.180456:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177215.180465:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177215.180468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177215.180475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177215.180480:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177215.180484:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177215.180487:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177215.180490:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177215.180493:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177215.500480:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306562304, at 4306562368 00000100:00000040:1.0:1786177215.500483:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306562304, at 4306562368 00000100:00000010:0.0:1786177215.500496:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62000. 00000100:00000010:1.0:1786177215.500501:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62600. 00010000:00000001:0.0:1786177215.564444:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177215.564450:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177215.564454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177215.564456:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177215.564458:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177215.564465:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177215.564466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177215.564467:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177215.564469:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177215.564479:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177215.564485:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177215.564489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177215.564493:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177215.564496:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177215.564499:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177215.564530:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177215.564534:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177215.564537:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177216.204509:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177216.204519:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177216.204523:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177216.204530:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177216.204536:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177216.204542:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177216.204546:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177216.204549:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177216.204555:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.588486:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177216.588495:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177216.588497:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177216.588498:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177216.588504:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177216.588506:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177216.588511:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177216.588512:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177216.588515:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177216.588517:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.588519:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177216.588523:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177216.588524:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177216.588527:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177216.588528:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177216.588627:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177216.588631:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177216.588634:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177216.716501:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177216.716514:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177216.716518:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177216.716524:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177216.716530:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177216.716540:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11891 00000400:00000200:2.0:1786177216.716553:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177216.716561:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177216.716565:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177216.716569:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716574:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177216.716577:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716579:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177216.716583:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177216.716585:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177216.716588:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177216.716591:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177216.716593:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177216.716595:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177216.716597:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716598:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177216.716600:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716601:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177216.716605:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177216.716607:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716608:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177216.716611:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177216.716614:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177216.716616:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.716618:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177216.716620:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177216.716630:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 00000020:00000040:2.0:1786177216.716633:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177216.716637:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177216.716639:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.716642:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177216.716645:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177216.716650:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177216.716659:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786177216.716666:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177216.716675:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177216.716684:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177216.716689:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177216.716693:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177216.716713:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177216.716716:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177216.716719:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177216.716720:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177216.716721:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177216.716724:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177216.716726:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11891 00000400:00000200:2.0:1786177216.716729:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177216.716731:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177216.716733:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177216.716735:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716736:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177216.716738:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716739:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177216.716740:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177216.716741:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177216.716742:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177216.716744:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177216.716745:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177216.716747:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177216.716749:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716750:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177216.716752:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716753:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177216.716754:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177216.716756:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716757:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177216.716758:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177216.716765:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177216.716766:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.716768:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177216.716773:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177216.716775:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.716777:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177216.716781:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000020:00000040:2.0:1786177216.716783:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177216.716785:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177216.716786:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.716788:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1786177216.716790:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1786177216.716790:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177216.716793:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177216.716804:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:1.0:1786177216.716805:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177216.716812:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1786177216.716814:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177216.716825:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1786177216.716831:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177216.716840:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177216.716848:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177216.716862:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.716864:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177216.716869:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177216.716880:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1786177216.716883:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177216.716885:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1786177216.716893:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1786177216.716894:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177216.716906:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1786177216.716907:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235589760:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177216.716914:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177216.716917:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1786177216.716918:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177216.716922:0:40394:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1786177216.716923:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177216.716926:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177216.716927:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11891 02000000:00000001:1.0:1786177216.716933:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1786177216.716939:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177216.716941:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177216.716943:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177216.716945:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716946:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177216.716948:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.716949:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177216.716950:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177216.716951:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177216.716952:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177216.716954:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 02000000:00000010:1.0:1786177216.716955:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae000. 00000020:00000001:2.0:1786177216.716956:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 02000000:00000001:1.0:1786177216.716961:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177216.716961:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177216.716975:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177216.716985:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177216.716986:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177216.716998:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:1.0:1786177216.716999:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000100:00000001:2.0:1786177216.717013:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177216.717018:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000010:1.0:1786177216.717019:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00080000:2.0:1786177216.717035:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000200:1.0:1786177216.717037:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235589760, portal 25 00000100:00000001:2.0:1786177216.717038:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.717041:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1786177216.717042:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177216.717045:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469626368 : -110371239925248 : ffff9b9e31736a00) 02000000:00000001:2.0:1786177216.717047:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177216.717050:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.717052:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177216.717057:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1786177216.717058:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1786177216.717068:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.717079:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177216.717091:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000100:00000001:1.0:1786177216.717092:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:2.0:1786177216.717101:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000040:1.0:1786177216.717108:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 02000000:00000001:2.0:1786177216.717110:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1786177216.717112:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 02000000:00000001:2.0:1786177216.717113:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.717121:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1786177216.717122:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235589760, offset 0 02000000:00000001:2.0:1786177216.717128:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177216.717131:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177216.717138:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 02000000:00000010:2.0:1786177216.717141:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab400. 00000400:00000200:1.0:1786177216.717162:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177216.717162:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177216.717171:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00100000:2.0:1786177216.717174:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177216.717178:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177216.717185:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177216.717193:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177216.717199:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.717205:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1786177216.717207:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426674:426674:256:4294967295] 192.168.202.132@tcp LPNI seq info [426636:426636:8:4294967295] 00000100:00000001:2.0:1786177216.717213:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177216.717214:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177216.717225:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177216.717234:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177216.717254:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177216.717257:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177216.717261:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1786177216.717264:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00080000:2.0:1786177216.717265:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177216.717268:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177216.717271:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177216.717275:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177216.717280:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11891 00000800:00000200:1.0:1786177216.717283:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:2.0:1786177216.717290:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:1.0:1786177216.717294:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000400:00000200:2.0:1786177216.717295:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177216.717305:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177216.717312:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:1.1:1786177216.717313:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177216.717327:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1786177216.717328:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000040:2.0:1786177216.717337:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000100:1.0:1786177216.717344:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177216.717345:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000010:1.0:1786177216.717348:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000020:00000001:2.0:1786177216.717352:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177216.717354:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000400:00000200:1.0:1786177216.717356:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177216.717387:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000020:00000001:2.0:1786177216.717392:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1786177216.717393:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000020:00000040:2.0:1786177216.717397:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000400:00000010:1.0:1786177216.717398:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000020:00000001:2.0:1786177216.717405:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:1.0:1786177216.717408:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1786177216.717409:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177216.717418:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000200:1.0:1786177216.717419:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177216.717427:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177216.717437:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.717442:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177216.717445:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1786177216.717446:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000040:1.0:1786177216.717453:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177216.717454:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177216.717463:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177216.717470:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177216.717476:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177216.717485:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.717492:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177216.717500:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1786177216.717500:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1786177216.717509:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:2.0:1786177216.717512:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.717516:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.717518:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177216.717526:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177216.717531:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306567425 00000100:00000010:2.0:1786177216.717531:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000100:00000010:1.0:1786177216.717542:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62600. 00000020:00000040:2.0:1786177216.717542:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177216.717552:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177216.717554:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1786177216.717556:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.717559:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177216.717561:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177216.717563:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306567425 02000000:00000001:2.0:1786177216.717564:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177216.717568:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62ea0. 00000100:00000001:1.0:1786177216.717572:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 02000000:00000010:2.0:1786177216.717572:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:1.0:1786177216.717583:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177216.717585:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.717601:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.717606:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177216.717607:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1786177216.717616:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177216/real 1786177216] req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177216.717655:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177216.717658:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177216.717663:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177216.717668:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177216.717672:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177216.717677:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1786177216.717678:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1786177216.717683:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177216.717687:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177216.717701:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177216.717708:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.717711:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177216.717715:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.717720:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177216.717724:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177216.717734:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177216.717757:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.717758:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1786177216.717768:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1786177216.717769:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177216.717790:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177216.717796:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177216.717796:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177216.717806:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1786177216.717806:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177216.717816:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177216.717816:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177216.717825:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1786177216.717832:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11891 00000100:00000001:1.0:1786177216.717833:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177216.717838:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:2.0:1786177216.717847:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177216.717850:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:2.0:1786177216.717855:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177216.717861:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00080000:2.0:1786177216.717862:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177216.717866:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177216.717870:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177216.717876:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177216.717877:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177216.717887:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177216.717896:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00100000:1.0:1786177216.717903:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235589760:192.168.202.132@tcp:250:kworker.0 00000020:00000001:2.0:1786177216.717903:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177216.717910:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000001:1.0:1786177216.717911:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.717914:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1786177216.717917:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1786177216.717918:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736a00 x1872930235589760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177216.717923:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:1.0:1786177216.717925:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177216.717930:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:2.0:1786177216.717931:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 02000000:00000010:1.0:1786177216.717933:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177216.717945:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177216.717945:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:1.0:1786177216.717950:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177216.717955:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177216.717962:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000040:2.0:1786177216.717962:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000010:1.0:1786177216.717966:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786177216.717973:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1786177216.717975:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177216.717978:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:2.0:1786177216.717982:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000010:1.0:1786177216.717983:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736a00. 00000100:00000001:2.0:1786177216.717992:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177216.717993:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177216.717996:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718004:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1786177216.718004:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177216.718019:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177216.718023:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177216.718023:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718030:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177216.718030:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1786177216.718039:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1786177216.718040:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177216.718041:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1786177216.718048:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718049:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1786177216.718055:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177216.718058:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000040:1.0:1786177216.718059:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1786177216.718067:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.718079:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718090:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000010:2.0:1786177216.718091:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000100:00000001:1.0:1786177216.718092:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177216.718107:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177216.718108:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177216.718114:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177216.718115:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177216.718122:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177216.718122:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.718136:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1786177216.718140:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235589888:192.168.202.132@tcp:38:kworker.0 02000000:00000001:2.0:1786177216.718141:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177216.718159:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718172:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:2.0:1786177216.718172:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 02000000:00000001:1.0:1786177216.718175:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177216.718178:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177216.718179:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177216.718181:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177216.718186:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae000. 00000100:00100000:2.0:1786177216.718188:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1786177216.718190:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177216.718206:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177216.718207:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1786177216.718212:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000100:00000001:2.0:1786177216.718213:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177216.718226:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000040:2.0:1786177216.718229:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177216.718234:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235589888, portal 10 00000100:00000001:1.0:1786177216.718240:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177216.718240:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718245:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469625472 : -110371239926144 : ffff9b9e31736680) 00000100:00000040:2.0:1786177216.718247:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11901) 00000100:00000040:1.0:1786177216.718263:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718274:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177216.718278:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177216.718282:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177216.718284:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235589888, offset 0 00000400:00000200:1.0:1786177216.718288:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177216.718291:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177216.718294:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177216.718297:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177216.718299:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177216.718301:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177216.718302:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177216.718305:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426675:426675:256:4294967295] 192.168.202.132@tcp LPNI seq info [426637:426637:8:4294967295] 00000400:00000200:1.0:1786177216.718307:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177216.718312:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177216.718315:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177216.718318:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177216.718320:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177216.718322:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786177216.718326:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177216.718328:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1786177216.718331:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177216.718332:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177216.718334:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177216.718337:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177216.718339:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177216.718340:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177216.718342:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177216.718345:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718351:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177216.718353:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718375:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718377:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177216.718380:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177216.718383:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177216.718385:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177216.718387:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306563589 00000100:00000001:1.0:1786177216.718389:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306567425 00000100:00000001:1.0:1786177216.718391:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306563589 00000100:00000001:1.0:1786177216.718392:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306567425 00000100:00000001:1.0:1786177216.718393:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177216.718395:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718396:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177216.718398:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177216/real 1786177216] req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177216.718404:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177216.718405:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177216.718406:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177216.718408:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177216.718409:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177216.718410:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177216.718412:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177216.718418:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718422:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177216.718423:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718424:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177216.718425:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177216.718427:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718431:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177216.718433:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718438:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177216.718440:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718441:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177216.718442:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718443:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177216.718445:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177216.718447:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177216.718449:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177216.718450:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177216.718453:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177216.718457:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235589888:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177216.718460:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177216.718463:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718467:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177216.718468:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177216.718470:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177216.718471:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177216.718472:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177216.718474:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590016:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177216.718476:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177216.718478:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177216.718480:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177216.718481:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177216.718483:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177216.718485:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177216.718487:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177216.718489:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177216.718491:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177216.718493:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235590016, portal 10 00000100:00000001:1.0:1786177216.718495:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177216.718496:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469623680 : -110371239927936 : ffff9b9e31735f80) 00000100:00000040:1.0:1786177216.718499:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718504:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177216.718505:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177216.718507:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177216.718508:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235590016, offset 0 00000400:00000200:1.0:1786177216.718511:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177216.718513:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177216.718515:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177216.718517:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177216.718519:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177216.718520:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177216.718522:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177216.718524:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426676:426676:256:4294967295] 192.168.202.132@tcp LPNI seq info [426638:426638:8:4294967295] 00000400:00000200:1.0:1786177216.718527:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177216.718531:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177216.718533:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177216.718535:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177216.718538:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177216.718540:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786177216.718542:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177216.718544:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1786177216.718547:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177216.718549:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177216.718550:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177216.718553:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177216.718555:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177216.718556:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177216.718558:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177216.718560:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718565:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177216.718567:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718571:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718572:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177216.718574:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177216.718576:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177216.718578:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177216.718580:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306563589 00000100:00000001:1.0:1786177216.718582:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306567425 00000100:00000001:1.0:1786177216.718583:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306563589 00000100:00000001:1.0:1786177216.718584:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306567425 00000100:00000001:1.0:1786177216.718585:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177216.718587:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718588:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177216.718590:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177216/real 1786177216] req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177216.718595:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177216.718596:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177216.718597:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177216.718599:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177216.718600:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177216.718601:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177216.718603:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177216.718608:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718612:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177216.718613:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718615:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177216.718616:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177216.718618:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718622:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177216.718624:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718628:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177216.718629:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718630:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177216.718631:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718632:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177216.718633:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177216.718635:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177216.718637:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177216.718638:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177216.718640:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177216.718645:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590016:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177216.718647:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177216.718649:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718654:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177216.718655:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177216.718657:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177216.718658:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177216.718659:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177216.718661:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590144:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177216.718664:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177216.718665:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177216.718666:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177216.718667:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177216.718669:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177216.718670:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177216.718672:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177216.718673:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177216.718675:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177216.718677:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235590144, portal 4 00000100:00000001:1.0:1786177216.718678:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177216.718679:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469628160 : -110371239923456 : ffff9b9e31737100) 00000100:00000040:1.0:1786177216.718683:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718687:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177216.718689:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177216.718690:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177216.718692:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235590144, offset 0 00000400:00000200:1.0:1786177216.718694:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177216.718696:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177216.718697:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177216.718699:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177216.718700:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177216.718702:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177216.718703:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177216.718706:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426677:426677:256:4294967295] 192.168.202.132@tcp LPNI seq info [426639:426639:8:4294967295] 00000400:00000200:1.0:1786177216.718708:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177216.718712:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177216.718716:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177216.718717:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177216.718720:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177216.718721:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786177216.718724:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177216.718726:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1786177216.718728:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177216.718729:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177216.718731:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177216.718733:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177216.718734:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177216.718735:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177216.718737:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177216.718739:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718744:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177216.718745:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718749:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718751:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177216.718752:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177216.718754:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177216.718756:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177216.718758:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306563589 00000100:00000001:1.0:1786177216.718759:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306567425 00000100:00000001:1.0:1786177216.718761:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306563589 00000100:00000001:1.0:1786177216.718762:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306567425 00000100:00000001:1.0:1786177216.718764:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177216.718765:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718766:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177216.718768:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177216/real 1786177216] req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177216.718774:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177216.718775:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177216.718776:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177216.718777:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177216.718778:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177216.718779:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177216.718781:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177216.718786:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718789:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177216.718790:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718791:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177216.718792:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177216.718794:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718798:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177216.718800:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718804:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177216.718806:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718807:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177216.718807:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718809:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177216.718810:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177216.718811:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177216.718814:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177216.718815:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177216.718818:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177216.718823:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590144:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177216.718825:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718827:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177216.718828:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736680 x1872930235589888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718832:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177216.718833:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177216.718834:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:1.0:1786177216.718836:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177216.718837:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177216.718838:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177216.718839:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177216.718840:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:1.0:1786177216.718842:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177216.718843:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177216.718844:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736680. 00000100:00000001:1.0:1786177216.718845:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177216.718846:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718847:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177216.718849:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735f80 x1872930235590016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718853:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177216.718854:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177216.718855:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177216.718857:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177216.718857:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177216.718858:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177216.718859:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177216.718860:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:1.0:1786177216.718861:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177216.718862:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177216.718863:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735f80. 00000100:00000001:1.0:1786177216.718865:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177216.718866:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718867:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177216.718869:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737100 x1872930235590144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718873:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177216.718873:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177216.718874:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177216.718875:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177216.718876:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177216.718877:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177216.718878:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177216.718878:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:1.0:1786177216.718880:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177216.718880:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177216.718881:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737100. 00000100:00000001:1.0:1786177216.718882:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177216.718883:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718884:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.718887:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177216.718888:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718889:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177216.718890:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177216.718891:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177216.718893:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718897:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177216.718898:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177216.718899:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177216.718900:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177216.718901:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177216.718903:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590272:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177216.718905:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177216.718907:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177216.718908:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177216.718909:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177216.718911:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177216.718912:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177216.718914:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177216.718916:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177216.718918:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177216.718919:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235590272, portal 4 00000100:00000001:1.0:1786177216.718921:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177216.718922:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469618304 : -110371239933312 : ffff9b9e31734a80) 00000100:00000040:1.0:1786177216.718926:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718930:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177216.718931:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177216.718933:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177216.718935:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235590272, offset 0 00000400:00000200:1.0:1786177216.718937:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177216.718939:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177216.718941:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177216.718942:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177216.718944:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177216.718946:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177216.718947:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177216.718949:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426678:426678:256:4294967295] 192.168.202.132@tcp LPNI seq info [426640:426640:8:4294967295] 00000400:00000200:1.0:1786177216.718952:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177216.718956:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177216.718958:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177216.718960:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177216.718963:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177216.718964:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1786177216.718967:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177216.718969:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1786177216.718971:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177216.718973:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177216.718975:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177216.718977:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177216.718979:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177216.718980:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177216.718981:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177216.718983:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718988:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177216.718990:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177241 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.718994:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.718995:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177216.718997:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177216.718999:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177216.719001:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177216.719002:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306563591 00000100:00000001:1.0:1786177216.719004:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306567425 00000100:00000001:1.0:1786177216.719005:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306563591 00000100:00000001:1.0:1786177216.719007:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306567425 00000100:00000001:1.0:1786177216.719008:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177216.719009:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.719010:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177216.719013:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177216/real 1786177216] req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177216.719017:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177216.719018:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177216.719018:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177216.719020:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177216.719021:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177216.719023:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177216.719025:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177216.719030:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.719034:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177216.719035:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.719036:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177216.719037:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177216.719039:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.719043:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177216.719045:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.719049:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177216.719050:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.719051:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177216.719052:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.719053:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177216.719054:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177216.719055:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177216.719057:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177216.719058:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177216.719061:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177216.719065:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590272:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177216.719068:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.719069:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177216.719071:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734a80 x1872930235590272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177241 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177216.719075:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177216.719076:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177216.719078:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:1.0:1786177216.719079:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177216.719080:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177216.719080:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177216.719082:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177216.719082:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:1.0:1786177216.719084:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177216.719085:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177216.719085:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734a80. 00000100:00000001:1.0:1786177216.719087:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177216.719088:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.719089:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177216.719090:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177216.719091:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177216.719093:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177216.719094:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177217.228418:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177217.228427:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177217.228430:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177217.228436:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177217.228443:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177217.228448:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177217.228451:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177217.228453:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177217.228457:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177217.612428:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177217.612434:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177217.612436:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177217.612441:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177217.612442:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177217.612445:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177217.612448:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177217.612450:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177217.612455:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177217.612457:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177217.612458:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177217.612459:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177217.612461:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177217.612467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177217.612469:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177217.612521:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177217.612524:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177217.612527:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177218.252465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177218.252473:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177218.252477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177218.252482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177218.252487:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177218.252490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177218.252493:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177218.252496:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177218.252499:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177218.636454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177218.636458:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177218.636462:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177218.636463:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177218.636466:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177218.636469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177218.636471:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177218.636472:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177218.636477:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177218.636480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177218.636482:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177218.636484:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177218.636485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177218.636487:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177218.636489:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177218.636567:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177218.636573:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177218.636577:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177219.276879:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177219.276888:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177219.276891:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177219.276896:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177219.276900:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177219.276905:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177219.276908:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177219.276911:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177219.276915:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177219.660424:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177219.660431:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177219.660433:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177219.660437:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177219.660439:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177219.660440:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177219.660444:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177219.660445:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177219.660450:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177219.660450:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177219.660453:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177219.660454:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177219.660461:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177219.660461:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177219.660465:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177219.660516:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177219.660521:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177219.660524:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177220.300737:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177220.300745:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177220.300747:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177220.300753:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177220.300759:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177220.300764:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177220.300767:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177220.300771:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177220.300776:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177220.620453:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306567425, at 4306567488 00000100:00000040:1.0:1786177220.620453:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306567425, at 4306567488 00000100:00000010:1.0:1786177220.620467:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ea0. 00000100:00000010:0.0:1786177220.620470:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62600. 00000100:00000001:1.0:1786177220.684461:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177220.684463:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177220.684469:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177220.684471:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177220.684475:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177220.684476:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177220.684480:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:1.0:1786177220.684482:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177220.684483:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177220.684488:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177220.684489:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177220.684493:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177220.684493:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177220.684497:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177220.684498:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177220.684501:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177220.684502:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177220.684505:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177220.684506:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177220.684510:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177220.684512:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177220.684571:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177220.684574:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177220.684576:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177221.324869:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177221.324882:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177221.324885:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177221.324893:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177221.324899:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177221.324904:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177221.324907:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177221.324912:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177221.324915:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.708472:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177221.708474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177221.708479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.708482:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177221.708484:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177221.708486:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177221.708487:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177221.708494:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177221.708495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177221.708496:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177221.708502:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177221.708506:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177221.708509:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177221.708513:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177221.708516:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177221.708551:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177221.708556:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177221.708559:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177221.836483:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177221.836492:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177221.836494:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177221.836497:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177221.836501:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177221.836507:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11896 00000400:00000200:1.0:1786177221.836521:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177221.836527:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177221.836530:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177221.836532:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836535:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177221.836537:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836539:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177221.836542:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177221.836543:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177221.836545:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177221.836548:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177221.836549:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177221.836551:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177221.836553:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836554:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177221.836556:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836557:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177221.836560:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177221.836562:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836564:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177221.836566:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177221.836569:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177221.836571:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.836572:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177221.836574:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177221.836581:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328baa00. 00000020:00000040:1.0:1786177221.836584:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177221.836587:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177221.836588:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.836591:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177221.836593:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177221.836597:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177221.836603:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af400. 00000100:00000001:1.0:1786177221.836608:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177221.836615:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177221.836624:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177221.836627:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177221.836631:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ebce398] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177221.836649:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177221.836651:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177221.836653:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177221.836655:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177221.836656:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177221.836658:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177221.836660:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11896 00000400:00000200:1.0:1786177221.836663:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177221.836666:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177221.836668:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177221.836671:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836673:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177221.836674:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836676:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177221.836677:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1786177221.836678:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177221.836681:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000100:00000001:2.0:1786177221.836683:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1786177221.836684:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1786177221.836688:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1786177221.836689:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177221.836692:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177221.836694:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177221.836698:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177221.836699:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836703:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177221.836707:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836711:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177221.836715:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1786177221.836716:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:2.0:1786177221.836718:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177221.836720:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:2.0:1786177221.836722:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1786177221.836724:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177221.836726:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177221.836727:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177221.836731:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177221.836733:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1786177221.836735:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235590400:192.168.202.132@tcp:250:kworker.0 02000000:00000001:1.0:1786177221.836737:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.836742:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177221.836743:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:1.0:1786177221.836747:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1786177221.836747:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177221.836750:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1786177221.836754:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177221.836754:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1786177221.836759:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177221.836760:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177221.836762:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177221.836765:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bad80. 00000400:00000200:2.0:1786177221.836769:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:1.0:1786177221.836771:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000010:2.0:1786177221.836774:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 02000000:00000001:1.0:1786177221.836777:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177221.836780:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177221.836781:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000001:1.0:1786177221.836784:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177221.836786:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177221.836788:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177221.836788:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235590400, portal 25 00000100:00000001:2.0:1786177221.836792:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000010:1.0:1786177221.836794:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae000. 00000100:00000001:2.0:1786177221.836794:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487992832 : -110371221558784 : ffff9b9e328baa00) 00000100:00000001:1.0:1786177221.836801:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177221.836802:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177221.836810:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177221.836811:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177221.836818:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177221.836823:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000001:1.0:1786177221.836824:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177221.836827:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177221.836827:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235590400, offset 0 00000100:00000040:1.0:1786177221.836836:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b748f214] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177221.836839:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836847:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177221.836851:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177221.836852:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177221.836856:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177221.836857:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177221.836860:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1786177221.836860:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:1.0:1786177221.836867:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177221.836870:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1786177221.836872:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11896 00000400:00000200:2.0:1786177221.836877:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177221.836878:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177221.836883:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177221.836884:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177221.836888:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177221.836888:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000040:1.0:1786177221.836894:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177221.836898:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426679:426679:256:4294967295] 192.168.202.132@tcp LPNI seq info [426641:426641:8:4294967295] 00000100:00000001:1.0:1786177221.836899:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177221.836904:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177221.836908:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177221.836910:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177221.836913:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177221.836914:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177221.836916:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:2.0:1786177221.836921:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177221.836926:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000020:00000040:1.0:1786177221.836927:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000800:00000010:2.0:1786177221.836930:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000020:00000001:1.0:1786177221.836931:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177221.836935:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177221.836940:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:2.0:1786177221.836940:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177221.836946:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1786177221.836947:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00000040:1.0:1786177221.836949:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836953:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177221.836957:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000200:2.1:1786177221.836957:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:1.0:1786177221.836962:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:2.1:1786177221.836963:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1786177221.836965:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177221.836969:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177221.836971:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177221.836975:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1786177221.836975:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177221.836979:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000010:2.0:1786177221.836981:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000008:00100000:1.0:1786177221.836982:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177221.836987:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.836988:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177221.836991:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000010:1.0:1786177221.836992:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bbb80. 00000020:00000040:1.0:1786177221.836997:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177221.837000:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177221.837002:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177221.837003:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1786177221.837004:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177221.837007:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177221.837009:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177221.837012:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 02000000:00000010:1.0:1786177221.837013:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af800. 00000400:00000010:2.0:1786177221.837017:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:1.0:1786177221.837018:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177221.837025:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177221.837026:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177221.837033:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177221.837037:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177221.837041:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177221.837048:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3691ff3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837048:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177221.837055:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177221.837060:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177221.837066:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786177221.837067:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177221.837072:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177221.837073:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177221.837076:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177221.837078:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177221.837079:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.837081:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177221.837087:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11896 00000400:00000200:2.0:1786177221.837088:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177221.837092:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306572544 00000400:00000200:1.0:1786177221.837105:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177221.837114:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000010:2.0:1786177221.837114:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290ce40. 00000100:00080000:1.0:1786177221.837119:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177221.837122:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177221.837122:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177221.837127:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177221.837129:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.837131:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177221.837131:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306572544 00000020:00000001:1.0:1786177221.837136:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000010:2.0:1786177221.837137:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c9c0. 00000020:00000040:1.0:1786177221.837138:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:2.0:1786177221.837141:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1786177221.837142:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177221.837146:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:2.0:1786177221.837147:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1786177221.837148:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:2.0:1786177221.837150:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177221.837151:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:2.0:1786177221.837154:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1786177221.837155:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.837158:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1786177221.837161:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177221/real 1786177221] req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177221.837162:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.837168:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177221.837173:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:1.0:1786177221.837179:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177221.837184:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177221.837188:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177221.837192:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1786177221.837194:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1786177221.837195:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1786177221.837197:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1786177221.837198:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:2.0:1786177221.837200:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000008:00100000:1.0:1786177221.837202:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000200:2.0:1786177221.837204:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000008:00000001:1.0:1786177221.837205:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177221.837208:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:1.0:1786177221.837209:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.837213:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000010:1.0:1786177221.837214:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8000. 00000100:00000200:2.0:1786177221.837218:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177221.837219:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177221.837224:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177221.837227:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.837231:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177221.837236:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1786177221.837236:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177221.837240:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.837243:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177221.837245:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1786177221.837247:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad800. 00000100:00000001:2.0:1786177221.837247:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177221.837249:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177221.837255:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177221.837255:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837265:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177221.837268:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1786177221.837274:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177221.837287:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177221.837290:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.837293:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:1.0:1786177221.837297:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ab5f277] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837299:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.837304:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177221.837306:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177221.837310:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.837311:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1786177221.837313:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786177221.837315:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177221.837317:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177221.837321:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177221.837322:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177221.837323:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177221.837327:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177221.837328:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00080000:1.0:1786177221.837332:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11896 00000100:00000001:2.0:1786177221.837333:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:1.0:1786177221.837340:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177221.837342:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177221.837345:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177221.837350:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177221.837356:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00100000:2.0:1786177221.837379:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235590400:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177221.837383:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.837385:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177221.837387:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328baa00 x1872930235590400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837392:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177221.837395:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177221.837397:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 00000100:00000001:1.0:1786177221.837403:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1786177221.837403:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177221.837407:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000040:1.0:1786177221.837408:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000020:00000040:2.0:1786177221.837409:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000001:1.0:1786177221.837415:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177221.837415:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177221.837418:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af400. 00000020:00000001:1.0:1786177221.837419:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177221.837422:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 02000000:00000001:2.0:1786177221.837423:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:1.0:1786177221.837424:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:2.0:1786177221.837426:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000040:1.0:1786177221.837427:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000010:2.0:1786177221.837429:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328baa00. 00000020:00000001:1.0:1786177221.837430:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177221.837433:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000001:1.0:1786177221.837434:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177221.837436:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177221.837440:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177221.837441:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177221.837443:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177221.837446:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177221.837447:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177221.837448:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:1.0:1786177221.837449:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:2.0:1786177221.837451:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177221.837454:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177221.837456:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177221.837459:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.837460:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177221.837462:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1786177221.837463:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1786177221.837467:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177221.837468:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177221.837474:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.837478:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177221.837481:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1786177221.837482:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1786177221.837485:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.837485:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.837491:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177221.837491:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177221.837495:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1786177221.837497:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9500. 02000000:00000001:2.0:1786177221.837498:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177221.837503:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00100000:2.0:1786177221.837504:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590528:192.168.202.132@tcp:38:kworker.0 02000000:00000001:1.0:1786177221.837508:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177221.837509:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177221.837510:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.837513:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177221.837513:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177221.837516:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177221.837516:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177221.837520:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1786177221.837521:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177221.837524:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000010:1.0:1786177221.837526:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af000. 02000000:00000001:2.0:1786177221.837527:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177221.837533:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177221.837535:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177221.837539:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00100000:1.0:1786177221.837540:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1786177221.837543:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177221.837548:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235590528, portal 10 00000100:00000001:2.0:1786177221.837551:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177221.837555:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487993728 : -110371221557888 : ffff9b9e328bad80) 00000100:00000001:1.0:1786177221.837561:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177221.837564:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177221.837568:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177221.837569:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000cd924b3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177221.837602:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.837603:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177221.837608:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11906) 00000100:00000040:2.0:1786177221.837609:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177221.837614:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177221.837616:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235590528, offset 0 00000400:00000200:2.0:1786177221.837620:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177221.837624:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177221.837627:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177221.837630:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177221.837632:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177221.837634:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177221.837636:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177221.837638:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426680:426680:256:4294967295] 192.168.202.132@tcp LPNI seq info [426642:426642:8:4294967295] 00000400:00000200:2.0:1786177221.837641:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177221.837645:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177221.837649:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177221.837651:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177221.837654:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177221.837657:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786177221.837660:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177221.837662:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1786177221.837665:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177221.837666:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177221.837668:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177221.837671:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177221.837673:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177221.837674:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177221.837676:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177221.837679:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837685:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177221.837687:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837691:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.837692:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177221.837694:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177221.837697:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177221.837699:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177221.837701:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306568708 00000100:00000001:2.0:1786177221.837703:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306572544 00000100:00000001:2.0:1786177221.837705:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306568708 00000100:00000001:2.0:1786177221.837706:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306572544 00000100:00000001:2.0:1786177221.837708:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177221.837709:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.837710:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177221.837713:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177221/real 1786177221] req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177221.837717:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177221.837718:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177221.837720:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177221.837721:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177221.837722:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177221.837724:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177221.837726:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177221.837731:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837735:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177221.837736:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.837737:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177221.837738:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177221.837740:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837744:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177221.837746:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837751:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177221.837753:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.837754:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177221.837755:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.837756:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177221.837757:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177221.837759:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177221.837761:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177221.837762:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177221.837764:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177221.837769:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590528:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177221.837772:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177221.837774:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837779:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177221.837780:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177221.837781:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177221.837782:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177221.837783:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177221.837785:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590656:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177221.837787:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177221.837788:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177221.837789:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177221.837790:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177221.837793:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177221.837794:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177221.837796:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177221.837798:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177221.837800:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177221.837801:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235590656, portal 10 00000100:00000001:2.0:1786177221.837803:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177221.837804:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487997312 : -110371221554304 : ffff9b9e328bbb80) 00000100:00000040:2.0:1786177221.837807:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837812:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177221.837813:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177221.837815:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177221.837816:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235590656, offset 0 00000400:00000200:2.0:1786177221.837818:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177221.837820:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177221.837822:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177221.837824:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177221.837826:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177221.837828:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177221.837829:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177221.837831:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426681:426681:256:4294967295] 192.168.202.132@tcp LPNI seq info [426643:426643:8:4294967295] 00000400:00000200:2.0:1786177221.837834:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177221.837838:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177221.837840:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177221.837842:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177221.837845:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177221.837846:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786177221.837849:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177221.837851:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1786177221.837853:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177221.837854:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177221.837856:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177221.837858:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177221.837860:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177221.837861:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177221.837863:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177221.837865:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837870:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177221.837872:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837876:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.837877:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177221.837879:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177221.837880:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177221.837883:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177221.837884:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306568708 00000100:00000001:2.0:1786177221.837885:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306572544 00000100:00000001:2.0:1786177221.837887:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306568708 00000100:00000001:2.0:1786177221.837889:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306572544 00000100:00000001:2.0:1786177221.837890:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177221.837891:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.837893:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177221.837895:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177221/real 1786177221] req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177221.837899:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177221.837900:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177221.837901:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177221.837902:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177221.837903:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177221.837904:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177221.837906:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177221.837912:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837916:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177221.837917:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.837918:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177221.837919:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177221.837921:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837925:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177221.837927:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837931:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177221.837932:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.837933:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177221.837934:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.837935:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177221.837937:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177221.837938:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177221.837940:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177221.837942:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177221.837944:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177221.837949:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590656:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177221.837951:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177221.837953:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.837957:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177221.837958:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177221.837959:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177221.837960:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177221.837961:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177221.837963:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590784:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177221.837966:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177221.837968:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177221.837970:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177221.837971:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177221.837974:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177221.837976:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177221.837980:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177221.837982:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177221.837985:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177221.837988:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235590784, portal 4 00000100:00000001:2.0:1786177221.837990:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177221.837991:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982080 : -110371221569536 : ffff9b9e328b8000) 00000100:00000040:2.0:1786177221.837998:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838004:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177221.838007:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177221.838009:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177221.838012:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235590784, offset 0 00000400:00000200:2.0:1786177221.838016:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177221.838019:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177221.838022:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177221.838024:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177221.838026:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177221.838027:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177221.838029:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177221.838031:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426682:426682:256:4294967295] 192.168.202.132@tcp LPNI seq info [426644:426644:8:4294967295] 00000400:00000200:2.0:1786177221.838034:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177221.838038:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177221.838041:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177221.838043:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177221.838046:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177221.838048:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786177221.838051:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177221.838053:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1786177221.838055:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177221.838056:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177221.838058:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177221.838061:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177221.838063:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177221.838064:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177221.838066:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177221.838068:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838074:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177221.838076:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838084:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.838085:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177221.838087:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177221.838089:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177221.838091:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177221.838092:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306568708 00000100:00000001:2.0:1786177221.838108:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306572544 00000100:00000001:2.0:1786177221.838110:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306568708 00000100:00000001:2.0:1786177221.838111:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306572544 00000100:00000001:2.0:1786177221.838113:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177221.838114:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.838116:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177221.838118:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177221/real 1786177221] req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177221.838123:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177221.838124:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177221.838126:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177221.838127:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177221.838128:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177221.838130:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177221.838132:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177221.838137:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838141:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177221.838142:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838143:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177221.838144:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177221.838146:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838151:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177221.838153:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838158:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177221.838159:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838160:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177221.838161:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838162:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177221.838163:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177221.838165:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177221.838166:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177221.838168:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177221.838170:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177221.838175:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590784:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177221.838177:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838178:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177221.838180:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bad80 x1872930235590528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838184:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177221.838185:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177221.838187:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177221.838188:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177221.838189:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177221.838190:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177221.838192:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177221.838193:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:2.0:1786177221.838195:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177221.838196:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177221.838197:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bad80. 00000100:00000001:2.0:1786177221.838200:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177221.838201:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838202:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177221.838204:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bbb80 x1872930235590656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838208:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177221.838209:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177221.838210:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177221.838211:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177221.838212:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177221.838213:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177221.838214:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177221.838215:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af800. 02000000:00000001:2.0:1786177221.838216:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177221.838216:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177221.838217:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bbb80. 00000100:00000001:2.0:1786177221.838219:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177221.838219:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838220:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177221.838222:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8000 x1872930235590784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838226:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177221.838226:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177221.838227:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177221.838228:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177221.838229:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177221.838230:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177221.838231:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177221.838232:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:2.0:1786177221.838233:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177221.838233:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177221.838234:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8000. 00000100:00000001:2.0:1786177221.838235:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177221.838237:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838238:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838240:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177221.838241:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.838243:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177221.838244:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177221.838245:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177221.838247:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838251:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177221.838252:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177221.838253:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177221.838254:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177221.838256:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177221.838258:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590912:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177221.838260:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177221.838261:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177221.838262:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177221.838263:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177221.838266:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177221.838268:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177221.838271:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177221.838274:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177221.838277:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177221.838280:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235590912, portal 4 00000100:00000001:2.0:1786177221.838282:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177221.838283:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487987456 : -110371221564160 : ffff9b9e328b9500) 00000100:00000040:2.0:1786177221.838289:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838296:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177221.838298:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177221.838301:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177221.838303:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235590912, offset 0 00000400:00000200:2.0:1786177221.838308:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177221.838312:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177221.838316:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177221.838320:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177221.838322:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177221.838324:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177221.838325:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177221.838328:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426683:426683:256:4294967295] 192.168.202.132@tcp LPNI seq info [426645:426645:8:4294967295] 00000400:00000200:2.0:1786177221.838331:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177221.838335:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177221.838338:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177221.838340:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177221.838343:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177221.838345:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1786177221.838348:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177221.838350:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1786177221.838352:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177221.838353:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177221.838370:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177221.838373:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177221.838375:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177221.838376:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177221.838378:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177221.838381:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838386:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177221.838388:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177246 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838393:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.838394:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177221.838396:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177221.838398:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177221.838400:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177221.838402:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306568711 00000100:00000001:2.0:1786177221.838403:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306572544 00000100:00000001:2.0:1786177221.838405:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306568711 00000100:00000001:2.0:1786177221.838407:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306572544 00000100:00000001:2.0:1786177221.838408:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177221.838409:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.838410:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177221.838412:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177221/real 1786177221] req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177221.838417:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177221.838418:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177221.838419:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177221.838421:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177221.838422:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177221.838423:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177221.838425:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177221.838430:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838434:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177221.838435:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838436:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177221.838437:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177221.838439:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838444:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177221.838446:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838450:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177221.838452:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838453:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177221.838454:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838455:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177221.838456:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177221.838458:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177221.838460:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177221.838461:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177221.838464:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177221.838468:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235590912:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177221.838470:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838472:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177221.838473:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9500 x1872930235590912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177246 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177221.838478:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177221.838479:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177221.838480:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177221.838481:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177221.838482:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177221.838483:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177221.838484:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177221.838485:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af000. 02000000:00000001:2.0:1786177221.838487:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177221.838487:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177221.838488:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9500. 00000100:00000001:2.0:1786177221.838490:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177221.838491:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838492:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177221.838494:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177221.838495:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177221.838497:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177221.838499:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177222.348927:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177222.348936:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177222.348939:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177222.348946:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177222.348952:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177222.348956:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177222.348959:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177222.348962:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177222.348965:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177222.732503:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177222.732509:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177222.732512:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177222.732517:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177222.732519:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177222.732524:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1786177222.732524:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177222.732531:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177222.732531:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177222.732536:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177222.732538:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177222.732541:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177222.732543:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177222.732545:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177222.732548:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177222.732550:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177222.732554:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177222.732561:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177222.732563:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177222.732567:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177222.732569:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177222.732571:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177222.732573:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177222.732580:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177222.732582:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177222.732584:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177222.732586:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177222.732643:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177222.732648:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177222.732651:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177223.376205:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177223.376214:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177223.376217:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177223.376231:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177223.376235:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177223.376239:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177223.376241:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177223.376244:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177223.376247:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177223.756473:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177223.756479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177223.756480:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177223.756481:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177223.756484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177223.756487:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177223.756490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177223.756491:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177223.756493:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177223.756495:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177223.756498:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177223.756499:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177223.756501:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177223.756502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177223.756504:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177223.756555:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177223.756557:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177223.756559:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177224.396809:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177224.396820:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177224.396822:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177224.396829:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177224.396834:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177224.396838:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177224.396841:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177224.396844:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177224.396848:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177224.780467:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177224.780469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177224.780473:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177224.780475:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177224.780477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177224.780480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177224.780480:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177224.780485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177224.780487:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177224.780488:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177224.780493:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177224.780497:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177224.780500:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177224.780502:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177224.780505:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177224.780540:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177224.780544:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177224.780546:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177225.420422:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177225.420428:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177225.420430:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177225.420434:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177225.420437:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177225.420440:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177225.420442:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177225.420444:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177225.420447:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177225.740456:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306572544, at 4306572608 00000100:00000040:1.0:1786177225.740456:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306572544, at 4306572608 00000100:00000010:1.0:1786177225.740468:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c9c0. 00000100:00000010:0.0:1786177225.740470:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290ce40. 00000100:00000001:1.0:1786177225.804426:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177225.804431:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177225.804433:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177225.804434:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177225.804435:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177225.804439:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177225.804439:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177225.804440:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177225.804442:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177225.804443:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177225.804444:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177225.804445:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177225.804448:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177225.804450:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177225.804453:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177225.804504:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177225.804507:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177225.804509:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177226.444445:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177226.444452:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177226.444455:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177226.444460:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177226.444464:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177226.444467:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.444470:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177226.444473:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.444486:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177226.828511:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177226.828514:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177226.828519:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.828522:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177226.828525:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177226.828527:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177226.828528:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177226.828534:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177226.828536:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177226.828536:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177226.828543:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177226.828547:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177226.828550:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177226.828554:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177226.828559:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177226.828615:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177226.828620:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177226.828624:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177226.956540:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177226.956555:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177226.956558:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177226.956563:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177226.956568:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177226.956577:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11901 00000400:00000200:1.0:1786177226.956586:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177226.956592:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177226.956595:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177226.956598:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.956601:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177226.956603:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.956605:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177226.956609:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177226.956610:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177226.956613:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177226.956616:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177226.956617:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177226.956619:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177226.956622:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.956623:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177226.956625:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.956626:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177226.956628:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177226.956631:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177226.956633:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177226.956635:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177226.956638:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177226.956640:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.956642:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177226.956644:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177226.956651:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9880. 00000020:00000040:1.0:1786177226.956653:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177226.956657:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177226.956658:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.956662:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177226.956664:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177226.956669:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177226.956677:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261aec00. 00000100:00000001:1.0:1786177226.956683:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177226.956692:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177226.956702:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177226.956706:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177226.956710:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2b2226e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177226.956731:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177226.956734:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177226.956737:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177226.956738:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177226.956740:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177226.956742:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177226.956743:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11901 00000400:00000200:1.0:1786177226.956746:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177226.956748:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177226.956750:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177226.956752:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.956754:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177226.956756:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.956758:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177226.956760:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177226.956761:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177226.956762:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177226.956763:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00000001:2.0:1786177226.956765:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1786177226.956766:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177226.956770:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177226.956773:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1786177226.956777:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177226.956778:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177226.956783:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177226.956788:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177226.956789:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.956796:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177226.956803:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:2.0:1786177226.956804:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177226.956807:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177226.956811:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:2.0:1786177226.956813:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1786177226.956818:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.956818:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177226.956824:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177226.956825:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1786177226.956829:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235591040:192.168.202.132@tcp:250:kworker.0 02000000:00000001:1.0:1786177226.956833:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177226.956834:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177226.956838:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.956839:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1786177226.956841:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:2.0:1786177226.956842:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.956846:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1786177226.956847:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177226.956851:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177226.956851:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000001:1.0:1786177226.956853:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.956854:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177226.956858:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8380. 00000400:00000200:2.0:1786177226.956862:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:1.0:1786177226.956863:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177226.956867:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:2.0:1786177226.956868:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 02000000:00000001:1.0:1786177226.956869:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.956874:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177226.956876:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177226.956877:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177226.956878:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 02000000:00000010:1.0:1786177226.956882:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af000. 00000100:00000200:2.0:1786177226.956887:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235591040, portal 25 00000100:00000001:1.0:1786177226.956888:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.956894:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177226.956897:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487988352 : -110371221563264 : ffff9b9e328b9880) 00000100:00100000:1.0:1786177226.956898:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1786177226.956907:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177226.956908:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177226.956913:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177226.956920:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002fbc72ce] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.956921:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177226.956929:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177226.956935:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000001:1.0:1786177226.956936:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177226.956939:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235591040, offset 0 00000100:00080000:1.0:1786177226.956940:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177226.956945:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177226.956946:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177226.956948:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1786177226.956951:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177226.956952:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177226.956956:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11901 00000400:00000200:1.0:1786177226.956959:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177226.956961:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177226.956963:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177226.956966:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.956971:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177226.956971:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177226.956977:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177226.956980:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:1.0:1786177226.956982:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177226.956988:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:2.0:1786177226.956988:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:1.0:1786177226.956992:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000400:00000200:2.0:1786177226.956995:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000001:1.0:1786177226.956998:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:2.0:1786177226.957001:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000020:00000040:1.0:1786177226.957004:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:2.0:1786177226.957005:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000001:1.0:1786177226.957010:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786177226.957011:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426684:426684:256:4294967295] 192.168.202.132@tcp LPNI seq info [426646:426646:8:4294967295] 00000020:00000001:1.0:1786177226.957017:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786177226.957020:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177226.957025:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.957027:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177226.957032:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.957036:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177226.957037:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000020:00000040:1.0:1786177226.957040:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:1.0:1786177226.957060:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177226.957062:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177226.957063:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177226.957066:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177226.957067:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.957069:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177226.957071:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.0:1786177226.957073:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000008:00000001:1.0:1786177226.957075:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.957078:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177226.957081:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9500. 00000020:00000040:1.0:1786177226.957086:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000800:00000010:2.0:1786177226.957086:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 02000000:00000001:1.0:1786177226.957092:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177226.957093:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.957094:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177226.957096:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:2.0:1786177226.957098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:1.0:1786177226.957099:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177226.957103:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 02000000:00000010:1.0:1786177226.957105:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad800. 00000100:00000001:1.0:1786177226.957112:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786177226.957113:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177226.957118:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00100000:1.0:1786177226.957119:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1786177226.957127:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177226.957132:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000100:00000001:1.0:1786177226.957135:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1786177226.957138:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1786177226.957140:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177226.957144:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:1.0:1786177226.957151:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000cd924b3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177226.957151:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177226.957155:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177226.957160:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177226.957168:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177226.957169:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177226.957175:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177226.957181:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177226.957185:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177226.957188:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177226.957196:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786177226.957197:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1786177226.957201:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11901 00000100:00000040:2.0:1786177226.957205:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177226.957210:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177226.957218:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177226.957221:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177226.957222:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177226.957226:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1786177226.957228:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.957231:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177226.957233:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177226.957234:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.957235:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177226.957240:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177226.957241:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000400:00000200:2.0:1786177226.957244:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:1.0:1786177226.957245:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1786177226.957248:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306577664 00000100:00000010:2.0:1786177226.957254:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c000. 00000020:00000040:1.0:1786177226.957256:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177226.957259:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177226.957262:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:2.0:1786177226.957264:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000040:1.0:1786177226.957267:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.957272:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177226.957274:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306577664 00000100:00000040:1.0:1786177226.957275:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.957280:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000010:2.0:1786177226.957282:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c300. 00000020:00000040:1.0:1786177226.957284:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:2.0:1786177226.957288:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1786177226.957290:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177226.957295:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177226.957296:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177226.957303:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177226.957303:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.957309:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1786177226.957310:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1786177226.957315:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177226/real 1786177226] req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177226.957316:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.957323:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177226.957328:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000001:2.0:1786177226.957330:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177226.957332:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:1.0:1786177226.957335:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177226.957336:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177226.957340:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000100:00000001:1.0:1786177226.957342:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177226.957343:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177226.957348:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000010:1.0:1786177226.957350:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8000. 00000100:00000200:2.0:1786177226.957351:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177226.957399:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.957406:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177226.957409:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177226.957410:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:2.0:1786177226.957411:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177226.957415:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177226.957417:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1786177226.957420:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177226.957421:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.957427:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177226.957430:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1786177226.957432:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177226.957434:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177226.957439:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af800. 00000100:00000040:2.0:1786177226.957444:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177226.957446:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.957455:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:1.0:1786177226.957459:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177226.957460:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.957463:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177226.957465:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.957469:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177226.957472:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177226.957477:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1786177226.957480:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00080000:2.0:1786177226.957483:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177226.957486:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.957487:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177226.957496:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177226.957500:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ab5f277] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177226.957506:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235591040:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177226.957514:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.957518:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1786177226.957522:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177226.957522:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9880 x1872930235591040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177226.957530:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786177226.957535:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:1.0:1786177226.957537:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177226.957541:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1786177226.957541:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1786177226.957545:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:2.0:1786177226.957546:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 00000100:00000001:1.0:1786177226.957551:0:191280:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1786177226.957553:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1786177226.957558:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11901 00000020:00000001:2.0:1786177226.957558:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177226.957562:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000400:00000200:1.0:1786177226.957568:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:2.0:1786177226.957568:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177226.957573:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261aec00. 00000400:00000200:1.0:1786177226.957576:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 02000000:00000001:2.0:1786177226.957580:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:1.0:1786177226.957582:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1786177226.957586:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:1.0:1786177226.957590:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000010:2.0:1786177226.957590:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9880. 00000100:00000001:2.0:1786177226.957594:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177226.957596:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177226.957597:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.957601:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177226.957604:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.957607:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177226.957610:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177226.957612:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177226.957613:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177226.957616:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:1.0:1786177226.957618:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177226.957618:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177226.957621:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177226.957624:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177226.957625:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177226.957628:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177226.957630:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177226.957632:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:2.0:1786177226.957637:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177226.957638:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177226.957644:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177226.957649:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177226.957655:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1786177226.957656:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:2.0:1786177226.957658:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.957662:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1786177226.957663:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:2.0:1786177226.957664:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177226.957668:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.957670:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177226.957673:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591168:192.168.202.132@tcp:38:kworker.0 02000000:00000001:1.0:1786177226.957675:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177226.957678:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177226.957680:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177226.957682:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177226.957682:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.957685:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1786177226.957689:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177226.957690:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000001:1.0:1786177226.957693:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:2.0:1786177226.957694:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1786177226.957698:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000200:2.0:1786177226.957704:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000008:00000001:1.0:1786177226.957706:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177226.957710:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177226.957711:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00000010:1.0:1786177226.957716:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bbb80. 00000400:00000010:2.0:1786177226.957718:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000020:00000040:1.0:1786177226.957720:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000200:2.0:1786177226.957725:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235591168, portal 10 02000000:00000001:1.0:1786177226.957726:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177226.957728:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1786177226.957729:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.957730:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982976 : -110371221568640 : ffff9b9e328b8380) 00000100:00000001:1.0:1786177226.957734:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177226.957737:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177226.957741:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177226.957741:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177226.957750:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad400. 00000100:00000001:2.0:1786177226.957751:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177226.957755:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177226.957758:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177226.957759:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177226.957763:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235591168, offset 0 00000400:00000200:2.0:1786177226.957769:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00100000:1.0:1786177226.957770:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786177226.957774:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177226.957780:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177226.957788:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177226.957792:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:1.0:1786177226.957794:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1786177226.957796:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1786177226.957801:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177226.957802:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177226.957807:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426685:426685:256:4294967295] 192.168.202.132@tcp LPNI seq info [426647:426647:8:4294967295] 00000100:00000040:1.0:1786177226.957813:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3691ff3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177226.957814:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177226.957825:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177226.957838:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177226.957844:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00000001:1.0:1786177226.957845:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177226.957850:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177226.957853:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000100:00000040:1.0:1786177226.957856:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11911) 00000800:00000200:2.1:1786177226.957857:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177226.957862:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1786177226.957865:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177226.957867:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177226.957869:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177226.957872:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177226.957874:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177226.957875:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177226.957877:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177226.957880:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.957886:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177226.957889:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.957899:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.957900:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177226.957903:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177226.957906:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177226.957908:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177226.957910:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306573828 00000100:00000001:2.0:1786177226.957912:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306577664 00000100:00000001:2.0:1786177226.957914:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306573828 00000100:00000001:2.0:1786177226.957915:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306577664 00000100:00000001:2.0:1786177226.957916:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177226.957918:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.957919:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177226.957921:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177226/real 1786177226] req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177226.957927:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177226.957928:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177226.957929:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177226.957930:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177226.957931:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177226.957933:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177226.957935:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177226.957940:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.957944:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177226.957945:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.957947:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177226.957948:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177226.957950:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.957956:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177226.957959:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.957963:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177226.957965:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.957966:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177226.957967:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.957968:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177226.957969:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177226.957971:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177226.957973:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177226.957974:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177226.957977:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177226.957981:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591168:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177226.957984:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177226.957986:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.957990:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177226.957991:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.957993:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177226.957994:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177226.957996:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177226.957998:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591296:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177226.958000:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177226.958001:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177226.958002:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.958004:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177226.958006:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177226.958008:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177226.958010:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177226.958011:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177226.958014:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177226.958015:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235591296, portal 10 00000100:00000001:2.0:1786177226.958017:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177226.958018:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487987456 : -110371221564160 : ffff9b9e328b9500) 00000100:00000040:2.0:1786177226.958022:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958027:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177226.958028:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177226.958030:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177226.958031:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235591296, offset 0 00000400:00000200:2.0:1786177226.958034:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177226.958036:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177226.958038:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177226.958040:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177226.958042:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177226.958043:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177226.958061:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177226.958064:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426686:426686:256:4294967295] 192.168.202.132@tcp LPNI seq info [426648:426648:8:4294967295] 00000400:00000200:2.0:1786177226.958069:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177226.958074:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177226.958077:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177226.958080:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177226.958083:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177226.958085:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786177226.958088:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177226.958089:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1786177226.958092:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177226.958094:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177226.958097:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177226.958099:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177226.958101:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177226.958103:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177226.958104:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177226.958107:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958112:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177226.958114:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958119:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.958120:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177226.958122:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177226.958125:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177226.958127:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177226.958128:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306573828 00000100:00000001:2.0:1786177226.958130:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306577664 00000100:00000001:2.0:1786177226.958131:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306573828 00000100:00000001:2.0:1786177226.958133:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306577664 00000100:00000001:2.0:1786177226.958135:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177226.958136:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.958137:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177226.958139:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177226/real 1786177226] req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177226.958144:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177226.958145:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177226.958146:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177226.958147:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177226.958149:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177226.958150:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177226.958157:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177226.958163:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958167:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177226.958168:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958169:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177226.958170:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177226.958172:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958176:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177226.958178:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958183:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177226.958184:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958185:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177226.958186:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958187:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177226.958188:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177226.958190:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177226.958192:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177226.958194:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177226.958196:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177226.958201:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591296:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177226.958204:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177226.958206:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958211:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177226.958212:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.958214:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177226.958215:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177226.958216:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177226.958218:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591424:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177226.958219:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177226.958221:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177226.958222:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.958223:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177226.958226:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177226.958227:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177226.958229:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177226.958231:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177226.958233:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177226.958235:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235591424, portal 4 00000100:00000001:2.0:1786177226.958237:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177226.958238:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982080 : -110371221569536 : ffff9b9e328b8000) 00000100:00000040:2.0:1786177226.958242:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958248:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177226.958250:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177226.958253:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177226.958255:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235591424, offset 0 00000400:00000200:2.0:1786177226.958260:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177226.958264:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177226.958267:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177226.958270:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177226.958273:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177226.958276:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177226.958278:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177226.958283:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426687:426687:256:4294967295] 192.168.202.132@tcp LPNI seq info [426649:426649:8:4294967295] 00000400:00000200:2.0:1786177226.958288:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177226.958295:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177226.958300:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177226.958303:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177226.958306:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177226.958308:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786177226.958311:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177226.958313:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1786177226.958316:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177226.958317:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177226.958319:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177226.958322:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177226.958324:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177226.958325:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177226.958327:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177226.958330:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958336:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177226.958338:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958343:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.958344:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177226.958346:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177226.958348:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177226.958350:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177226.958352:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306573828 00000100:00000001:2.0:1786177226.958354:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306577664 00000100:00000001:2.0:1786177226.958370:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306573828 00000100:00000001:2.0:1786177226.958372:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306577664 00000100:00000001:2.0:1786177226.958374:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177226.958376:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.958378:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177226.958381:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177226/real 1786177226] req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177226.958387:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177226.958388:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177226.958389:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177226.958391:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177226.958392:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177226.958393:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177226.958395:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177226.958401:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958405:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177226.958406:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958407:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177226.958408:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177226.958410:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958414:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177226.958416:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958420:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177226.958422:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958423:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177226.958424:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958426:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177226.958427:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177226.958429:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177226.958432:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177226.958433:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177226.958436:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177226.958441:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591424:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177226.958443:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958444:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177226.958446:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8380 x1872930235591168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958450:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177226.958451:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177226.958453:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177226.958454:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177226.958455:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177226.958456:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177226.958458:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177226.958459:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af000. 02000000:00000001:2.0:1786177226.958461:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177226.958462:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177226.958463:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8380. 00000100:00000001:2.0:1786177226.958465:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177226.958466:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958467:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177226.958471:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9500 x1872930235591296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958478:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177226.958479:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177226.958481:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177226.958483:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177226.958485:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177226.958486:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177226.958488:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177226.958490:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:2.0:1786177226.958492:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177226.958493:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177226.958495:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9500. 00000100:00000001:2.0:1786177226.958498:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177226.958499:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958500:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177226.958503:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8000 x1872930235591424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958507:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177226.958508:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177226.958509:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177226.958511:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177226.958511:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177226.958512:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177226.958513:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177226.958514:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af800. 02000000:00000001:2.0:1786177226.958515:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177226.958516:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177226.958516:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8000. 00000100:00000001:2.0:1786177226.958518:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177226.958518:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958520:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958522:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177226.958523:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.958524:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177226.958525:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177226.958526:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177226.958528:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958535:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177226.958537:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.958539:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177226.958540:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177226.958542:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177226.958545:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591552:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177226.958549:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177226.958551:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177226.958553:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177226.958555:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177226.958558:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177226.958560:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177226.958565:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177226.958568:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177226.958571:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177226.958575:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235591552, portal 4 00000100:00000001:2.0:1786177226.958577:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177226.958578:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487997312 : -110371221554304 : ffff9b9e328bbb80) 00000100:00000040:2.0:1786177226.958585:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958590:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177226.958591:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177226.958593:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177226.958595:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235591552, offset 0 00000400:00000200:2.0:1786177226.958598:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177226.958601:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177226.958604:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177226.958606:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177226.958608:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177226.958610:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177226.958611:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177226.958613:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426688:426688:256:4294967295] 192.168.202.132@tcp LPNI seq info [426650:426650:8:4294967295] 00000400:00000200:2.0:1786177226.958616:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177226.958621:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177226.958624:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177226.958626:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177226.958629:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177226.958631:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1786177226.958634:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177226.958636:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1786177226.958639:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177226.958641:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177226.958643:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177226.958645:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177226.958647:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177226.958648:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177226.958650:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177226.958652:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958657:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177226.958659:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177251 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958664:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.958665:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177226.958667:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177226.958669:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177226.958671:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177226.958673:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306573831 00000100:00000001:2.0:1786177226.958674:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306577664 00000100:00000001:2.0:1786177226.958676:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306573831 00000100:00000001:2.0:1786177226.958677:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306577664 00000100:00000001:2.0:1786177226.958679:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177226.958680:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.958681:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177226.958684:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177226/real 1786177226] req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177226.958688:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177226.958689:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177226.958690:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177226.958692:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177226.958693:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177226.958695:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177226.958697:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177226.958702:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958706:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177226.958706:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958707:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177226.958708:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177226.958710:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958714:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177226.958717:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958721:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177226.958723:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958724:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177226.958725:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958726:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177226.958727:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177226.958729:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177226.958731:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177226.958732:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177226.958734:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177226.958739:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591552:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177226.958742:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958743:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177226.958745:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bbb80 x1872930235591552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177251 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177226.958749:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177226.958750:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177226.958752:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177226.958753:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177226.958754:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177226.958754:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177226.958755:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177226.958757:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:2.0:1786177226.958758:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177226.958759:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177226.958760:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bbb80. 00000100:00000001:2.0:1786177226.958763:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177226.958764:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958765:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177226.958768:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177226.958769:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177226.958770:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177226.958771:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177227.468489:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177227.468497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177227.468501:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177227.468506:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177227.468511:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177227.468519:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177227.468522:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177227.468525:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177227.468528:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177227.852478:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177227.852486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177227.852487:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177227.852489:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177227.852492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177227.852497:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177227.852498:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177227.852501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177227.852503:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177227.852505:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177227.852508:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177227.852508:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177227.852515:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177227.852518:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177227.852521:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177227.852576:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177227.852580:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177227.852583:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177228.492433:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177228.492442:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177228.492445:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177228.492458:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177228.492463:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177228.492467:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177228.492470:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177228.492473:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177228.492477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177228.876461:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177228.876470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177228.876472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177228.876473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177228.876478:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177228.876481:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177228.876485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177228.876487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177228.876489:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177228.876492:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177228.876495:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177228.876497:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177228.876499:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177228.876501:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177228.876507:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177228.876570:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177228.876576:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177228.876580:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177229.516463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177229.516472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177229.516475:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177229.516480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177229.516484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177229.516488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177229.516491:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177229.516494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177229.516497:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177229.900457:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177229.900460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177229.900465:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177229.900467:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177229.900470:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177229.900474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177229.900477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177229.900481:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177229.900483:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177229.900486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177229.900488:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177229.900492:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177229.900494:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177229.900496:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177229.900498:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177229.900546:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177229.900550:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177229.900553:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177230.540482:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177230.540494:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177230.540496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177230.540501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177230.540504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177230.540507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177230.540509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177230.540511:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177230.540513:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177230.860469:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306577664, at 4306577728 00000100:00000040:1.0:1786177230.860473:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306577664, at 4306577728 00000100:00000010:0.0:1786177230.860484:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c000. 00000100:00000010:1.0:1786177230.860487:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c300. 00000100:00000001:1.0:1786177230.924452:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177230.924458:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177230.924460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177230.924460:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177230.924466:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177230.924467:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177230.924473:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177230.924476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177230.924477:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177230.924481:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177230.924483:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177230.924484:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177230.924488:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177230.924489:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177230.924495:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177230.924500:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177230.924503:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177230.924506:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177230.924521:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177230.924525:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177230.924527:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177230.924587:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177230.924590:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177230.924593:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177231.564424:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177231.564431:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177231.564433:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177231.564438:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177231.564442:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177231.564445:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177231.564448:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177231.564451:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177231.564454:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177231.948418:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177231.948434:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177231.948437:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177231.948443:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177231.948448:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177231.948452:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177231.948454:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177231.948456:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177231.948457:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177231.948460:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177231.948462:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177231.948464:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177231.948465:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177231.948467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177231.948470:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177231.948522:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177231.948526:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177231.948529:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177232.076484:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177232.076493:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177232.076502:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177232.076507:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177232.076512:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177232.076521:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11906 00000400:00000200:1.0:1786177232.076531:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177232.076539:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177232.076544:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177232.076548:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.076551:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177232.076553:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.076555:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177232.076564:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177232.076566:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177232.076568:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177232.076571:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177232.076578:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177232.076581:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177232.076584:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.076586:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177232.076589:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.076591:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177232.076594:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177232.076598:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177232.076603:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177232.076606:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177232.076609:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177232.076612:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.076615:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177232.076618:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177232.076629:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bad80. 00000020:00000040:1.0:1786177232.076634:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177232.076638:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177232.076641:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.076644:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177232.076647:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177232.076652:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177232.076661:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae000. 00000100:00000001:1.0:1786177232.076667:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177232.076678:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177232.076686:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177232.076690:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177232.076693:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b748f214] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177232.076708:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177232.076711:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177232.076714:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177232.076715:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177232.076716:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177232.076719:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177232.076721:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11906 00000400:00000200:1.0:1786177232.076724:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177232.076728:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177232.076730:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177232.076733:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.076735:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177232.076737:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177232.076737:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177232.076740:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177232.076743:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177232.076745:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000100:00000001:2.0:1786177232.076745:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1786177232.076749:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177232.076751:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00000001:2.0:1786177232.076751:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1786177232.076755:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177232.076757:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177232.076762:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177232.076763:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.076767:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177232.076771:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.076776:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177232.076780:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:2.0:1786177232.076783:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1786177232.076784:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177232.076786:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.076787:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177232.076790:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177232.076791:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177232.076794:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177232.076795:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177232.076797:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177232.076800:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177232.076803:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235591680:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177232.076804:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177232.076810:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1786177232.076810:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:1.0:1786177232.076815:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177232.076815:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1786177232.076817:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177232.076819:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177232.076822:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177232.076826:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177232.076828:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177232.076829:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328baa00. 00000020:00000040:1.0:1786177232.076833:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177232.076835:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1786177232.076836:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177232.076837:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.076841:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1786177232.076841:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 02000000:00000001:1.0:1786177232.076842:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177232.076845:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177232.076851:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 02000000:00000010:1.0:1786177232.076852:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af400. 00000100:00000001:1.0:1786177232.076858:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177232.076860:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235591680, portal 25 00000100:00100000:1.0:1786177232.076865:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177232.076865:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177232.076868:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487993728 : -110371221557888 : ffff9b9e328bad80) 00000100:00000001:1.0:1786177232.076878:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1786177232.076878:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177232.076883:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177232.076891:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ebce398] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.076892:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177232.076899:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177232.076903:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000001:1.0:1786177232.076905:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177232.076907:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235591680, offset 0 00000100:00080000:1.0:1786177232.076910:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177232.076913:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:2.0:1786177232.076915:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177232.076916:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177232.076919:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177232.076922:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177232.076926:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11906 00000400:00000200:2.0:1786177232.076926:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177232.076932:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177232.076935:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177232.076936:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177232.076941:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177232.076943:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1786177232.076947:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.076951:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177232.076951:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:1.0:1786177232.076956:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177232.076958:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1786177232.076959:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177232.076962:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000001:1.0:1786177232.076963:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177232.076966:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177232.076970:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:2.0:1786177232.076970:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426689:426689:256:4294967295] 192.168.202.132@tcp LPNI seq info [426651:426651:8:4294967295] 00000020:00000040:1.0:1786177232.076973:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177232.076977:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786177232.076978:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:1.0:1786177232.076981:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177232.076985:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.076990:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177232.077010:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000040:1.0:1786177232.077018:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:2.0:1786177232.077021:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177232.077022:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177232.077027:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000010:2.0:1786177232.077031:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00080000:1.0:1786177232.077037:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177232.077041:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177232.077043:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:1.0:1786177232.077044:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177232.077047:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177232.077050:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177232.077051:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1786177232.077052:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177232.077055:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177232.077059:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786177232.077060:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177232.077062:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786177232.077064:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000100:00000010:1.0:1786177232.077067:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8a80. 00000800:00000100:2.0:1786177232.077071:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000020:00000040:1.0:1786177232.077073:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000800:00000010:2.0:1786177232.077075:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 02000000:00000001:1.0:1786177232.077076:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177232.077079:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.077080:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177232.077081:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1786177232.077082:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1786177232.077084:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177232.077088:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 02000000:00000010:1.0:1786177232.077089:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261afc00. 00000400:00000200:2.0:1786177232.077092:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000100:00000001:1.0:1786177232.077093:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177232.077096:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00100000:1.0:1786177232.077101:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177232.077102:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177232.077109:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177232.077112:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177232.077116:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177232.077124:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002bb61190] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077126:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177232.077132:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177232.077137:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177232.077144:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786177232.077145:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.077148:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1786177232.077152:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177232.077154:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.077155:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177232.077159:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1786177232.077160:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177232.077164:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306582784 00000100:00000001:1.0:1786177232.077166:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000010:2.0:1786177232.077170:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c300. 00000100:00080000:1.0:1786177232.077173:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11906 00000100:00000001:2.0:1786177232.077178:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000400:00000200:1.0:1786177232.077181:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177232.077185:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306582784 00000400:00000200:1.0:1786177232.077188:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000010:2.0:1786177232.077190:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c000. 00000100:00080000:1.0:1786177232.077194:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177232.077194:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177232.077201:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1786177232.077202:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177232.077204:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.077205:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177232.077207:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1786177232.077210:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.077215:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00100000:2.0:1786177232.077216:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177232/real 1786177232] req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177232.077220:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177232.077223:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177232.077226:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177232.077231:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000400:00000001:2.0:1786177232.077232:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177232.077235:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1786177232.077236:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000010:2.0:1786177232.077237:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177232.077241:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000020:00000001:1.0:1786177232.077244:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000010:2.0:1786177232.077244:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177232.077247:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177232.077254:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177232.077255:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.077262:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177232.077271:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:2.0:1786177232.077271:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177232.077279:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177232.077282:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177232.077285:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177232.077286:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:2.0:1786177232.077288:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177232.077290:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1786177232.077293:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00100000:2.0:1786177232.077297:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177232.077298:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177232.077302:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177232.077306:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177232.077308:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177232.077310:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.077315:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:2.0:1786177232.077315:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1786177232.077319:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177232.077324:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.077329:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.077331:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000010:1.0:1786177232.077333:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9180. 00000100:00000001:2.0:1786177232.077334:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.077339:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:1.0:1786177232.077340:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:2.0:1786177232.077342:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177232.077345:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177232.077346:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:1.0:1786177232.077348:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.077349:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177232.077353:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:2.0:1786177232.077354:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1786177232.077381:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:2.0:1786177232.077381:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177232.077385:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:1.0:1786177232.077386:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177232.077390:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177232.077391:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad000. 00000100:00000001:1.0:1786177232.077397:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177232.077402:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235591680:192.168.202.132@tcp:250:kworker.0 00000100:00100000:1.0:1786177232.077406:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177232.077408:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.077414:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1786177232.077418:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177232.077420:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177232.077426:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d01dc3b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177232.077427:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bad80 x1872930235591680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077446:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177232.077449:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177232.077453:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 00000100:00000001:1.0:1786177232.077454:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177232.077456:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177232.077459:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000100:00080000:1.0:1786177232.077460:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000040:2.0:1786177232.077461:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177232.077466:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177232.077469:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae000. 00000100:00080000:1.0:1786177232.077470:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177232.077475:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1786177232.077475:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177232.077478:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:1.0:1786177232.077480:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:2.0:1786177232.077481:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bad80. 00000100:00000001:1.0:1786177232.077486:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786177232.077488:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00080000:1.0:1786177232.077491:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11906 00000100:00000001:2.0:1786177232.077491:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.077494:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1786177232.077498:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177232.077502:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:1.0:1786177232.077504:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177232.077505:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.077509:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1786177232.077511:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177232.077512:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177232.077515:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177232.077519:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:2.0:1786177232.077520:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177232.077526:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177232.077533:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177232.077535:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177232.077537:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177232.077539:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177232.077542:0:191280:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:2.0:1786177232.077544:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1786177232.077546:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 02000000:00000001:2.0:1786177232.077548:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1786177232.077549:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177232.077553:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 02000000:00000001:2.0:1786177232.077554:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177232.077557:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:1.0:1786177232.077559:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:1.0:1786177232.077561:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.077563:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177232.077564:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177232.077566:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177232.077569:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00100000:2.0:1786177232.077571:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591808:192.168.202.132@tcp:38:kworker.0 00000100:00080000:1.0:1786177232.077573:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177232.077577:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177232.077581:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177232.077582:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177232.077584:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177232.077585:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177232.077588:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177232.077589:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177232.077591:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1786177232.077593:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000010:2.0:1786177232.077595:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 00000008:00100000:1.0:1786177232.077597:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 02000000:00000001:2.0:1786177232.077599:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1786177232.077601:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.077604:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177232.077607:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000010:1.0:1786177232.077609:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb800. 00000400:00000010:2.0:1786177232.077611:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000020:00000040:1.0:1786177232.077613:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177232.077616:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:2.0:1786177232.077616:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 02000000:00000001:1.0:1786177232.077619:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.077622:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177232.077623:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177232.077624:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177232.077628:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261acc00. 00000100:00000200:2.0:1786177232.077629:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235591808, portal 10 00000100:00000001:1.0:1786177232.077631:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.077635:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1786177232.077638:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177232.077639:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487992832 : -110371221558784 : ffff9b9e328baa00) 00000100:00000001:1.0:1786177232.077649:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1786177232.077650:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177232.077652:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177232.077660:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008148c2a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077666:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1786177232.077669:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177232.077672:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000040:1.0:1786177232.077675:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11916) 00000400:00000010:2.0:1786177232.077676:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177232.077681:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235591808, offset 0 00000400:00000200:2.0:1786177232.077685:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177232.077689:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177232.077692:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177232.077695:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177232.077697:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177232.077699:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177232.077701:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177232.077705:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426690:426690:256:4294967295] 192.168.202.132@tcp LPNI seq info [426652:426652:8:4294967295] 00000400:00000200:2.0:1786177232.077710:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177232.077717:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177232.077721:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177232.077725:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177232.077730:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177232.077733:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786177232.077738:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177232.077741:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1786177232.077746:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177232.077748:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177232.077751:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177232.077755:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177232.077758:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177232.077760:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177232.077762:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177232.077765:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077771:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177232.077773:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077778:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.077779:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177232.077781:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177232.077784:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177232.077787:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177232.077788:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306578948 00000100:00000001:2.0:1786177232.077790:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306582784 00000100:00000001:2.0:1786177232.077792:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306578948 00000100:00000001:2.0:1786177232.077793:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306582784 00000100:00000001:2.0:1786177232.077795:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177232.077797:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.077798:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177232.077801:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177232/real 1786177232] req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177232.077806:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177232.077807:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177232.077808:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177232.077810:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177232.077811:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177232.077812:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177232.077814:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177232.077820:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077824:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177232.077825:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.077826:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177232.077827:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177232.077829:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077833:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177232.077835:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077839:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177232.077841:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.077842:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177232.077843:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.077844:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177232.077846:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177232.077848:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177232.077851:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177232.077852:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177232.077855:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177232.077860:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591808:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177232.077862:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177232.077864:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077868:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177232.077869:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177232.077871:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177232.077872:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177232.077873:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177232.077875:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591936:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177232.077877:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177232.077878:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177232.077880:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177232.077881:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177232.077883:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177232.077885:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177232.077887:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177232.077888:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177232.077890:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177232.077892:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235591936, portal 10 00000100:00000001:2.0:1786177232.077894:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177232.077895:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487984768 : -110371221566848 : ffff9b9e328b8a80) 00000100:00000040:2.0:1786177232.077898:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077903:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177232.077905:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177232.077906:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177232.077908:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235591936, offset 0 00000400:00000200:2.0:1786177232.077910:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177232.077913:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177232.077917:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177232.077920:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177232.077923:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177232.077925:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177232.077928:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177232.077931:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426691:426691:256:4294967295] 192.168.202.132@tcp LPNI seq info [426653:426653:8:4294967295] 00000400:00000200:2.0:1786177232.077936:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177232.077943:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177232.077948:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177232.077951:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177232.077954:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177232.077956:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786177232.077959:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177232.077961:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1786177232.077963:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177232.077965:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177232.077967:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177232.077969:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177232.077972:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177232.077973:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177232.077975:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177232.077977:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077983:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177232.077985:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.077989:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.077991:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177232.077993:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177232.078008:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177232.078013:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177232.078015:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306578948 00000100:00000001:2.0:1786177232.078017:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306582784 00000100:00000001:2.0:1786177232.078019:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306578948 00000100:00000001:2.0:1786177232.078021:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306582784 00000100:00000001:2.0:1786177232.078024:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177232.078026:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.078028:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177232.078033:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177232/real 1786177232] req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177232.078041:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177232.078043:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177232.078044:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177232.078047:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177232.078048:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177232.078050:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177232.078052:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177232.078058:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078063:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177232.078064:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078065:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177232.078066:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177232.078068:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078072:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177232.078075:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078080:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177232.078082:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078083:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177232.078083:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078085:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177232.078086:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177232.078088:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177232.078089:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177232.078091:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177232.078093:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177232.078098:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235591936:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177232.078101:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177232.078103:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078107:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177232.078108:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177232.078110:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177232.078111:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177232.078112:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177232.078114:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592064:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177232.078116:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177232.078117:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177232.078118:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177232.078120:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177232.078122:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177232.078123:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177232.078126:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177232.078128:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177232.078130:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177232.078132:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235592064, portal 4 00000100:00000001:2.0:1786177232.078134:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177232.078135:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487986560 : -110371221565056 : ffff9b9e328b9180) 00000100:00000040:2.0:1786177232.078139:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078143:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177232.078145:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177232.078146:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177232.078148:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235592064, offset 0 00000400:00000200:2.0:1786177232.078150:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177232.078153:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177232.078155:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177232.078158:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177232.078160:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177232.078162:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177232.078163:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177232.078165:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426692:426692:256:4294967295] 192.168.202.132@tcp LPNI seq info [426654:426654:8:4294967295] 00000400:00000200:2.0:1786177232.078168:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177232.078172:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177232.078175:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177232.078177:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177232.078180:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177232.078182:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786177232.078185:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177232.078186:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1786177232.078189:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177232.078190:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177232.078192:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177232.078195:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177232.078197:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177232.078198:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177232.078200:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177232.078202:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078207:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177232.078209:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078214:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.078215:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177232.078217:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177232.078219:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177232.078221:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177232.078222:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306578948 00000100:00000001:2.0:1786177232.078224:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306582784 00000100:00000001:2.0:1786177232.078226:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306578948 00000100:00000001:2.0:1786177232.078227:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306582784 00000100:00000001:2.0:1786177232.078228:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177232.078229:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.078230:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177232.078233:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177232/real 1786177232] req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177232.078237:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177232.078238:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177232.078239:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177232.078241:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177232.078242:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177232.078243:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177232.078245:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177232.078250:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078254:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177232.078255:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078257:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177232.078258:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177232.078260:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078264:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177232.078266:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078270:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177232.078271:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078272:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177232.078273:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078274:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177232.078275:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177232.078277:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177232.078278:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177232.078280:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177232.078282:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177232.078286:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592064:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177232.078289:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078290:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177232.078292:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328baa00 x1872930235591808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078296:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177232.078298:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177232.078299:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177232.078301:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177232.078302:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177232.078303:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177232.078305:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177232.078306:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af400. 02000000:00000001:2.0:1786177232.078308:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177232.078309:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177232.078310:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328baa00. 00000100:00000001:2.0:1786177232.078312:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177232.078313:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078314:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177232.078316:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8a80 x1872930235591936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078320:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177232.078321:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177232.078322:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177232.078323:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177232.078324:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177232.078325:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177232.078326:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177232.078327:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:2.0:1786177232.078328:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177232.078329:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177232.078329:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8a80. 00000100:00000001:2.0:1786177232.078331:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177232.078331:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078332:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177232.078334:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9180 x1872930235592064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078338:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177232.078339:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177232.078339:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177232.078341:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177232.078341:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177232.078342:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177232.078344:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177232.078344:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:2.0:1786177232.078346:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177232.078347:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177232.078347:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9180. 00000100:00000001:2.0:1786177232.078349:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177232.078350:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078351:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078353:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177232.078354:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.078371:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177232.078372:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177232.078373:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177232.078376:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078380:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177232.078381:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177232.078383:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177232.078384:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177232.078385:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177232.078387:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592192:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177232.078389:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177232.078391:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177232.078392:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177232.078393:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177232.078395:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177232.078396:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177232.078398:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177232.078400:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000400:00000010:2.0:1786177232.078402:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177232.078404:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235592192, portal 4 00000100:00000001:2.0:1786177232.078405:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177232.078406:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487996416 : -110371221555200 : ffff9b9e328bb800) 00000100:00000040:2.0:1786177232.078410:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078418:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177232.078420:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177232.078421:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00000200:2.0:1786177232.078423:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235592192, offset 0 00000400:00000200:2.0:1786177232.078425:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177232.078428:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177232.078430:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177232.078432:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177232.078433:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177232.078435:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177232.078436:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177232.078438:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426693:426693:256:4294967295] 192.168.202.132@tcp LPNI seq info [426655:426655:8:4294967295] 00000400:00000200:2.0:1786177232.078441:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177232.078445:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177232.078448:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177232.078450:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000800:00000200:2.0:1786177232.078453:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177232.078455:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1786177232.078457:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177232.078459:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1786177232.078461:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177232.078463:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:2.0:1786177232.078465:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177232.078467:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177232.078469:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:2.0:1786177232.078470:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:2.0:1786177232.078472:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177232.078474:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078479:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177232.078481:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078490:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.078491:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177232.078493:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177232.078494:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177232.078497:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177232.078498:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306578951 00000100:00000001:2.0:1786177232.078500:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306582784 00000100:00000001:2.0:1786177232.078501:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306578951 00000100:00000001:2.0:1786177232.078503:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306582784 00000100:00000001:2.0:1786177232.078504:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177232.078505:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.078506:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177232.078509:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177232/real 1786177232] req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177232.078513:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177232.078514:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177232.078515:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177232.078516:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177232.078517:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177232.078519:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177232.078521:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177232.078526:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078530:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177232.078530:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078531:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177232.078532:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177232.078534:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078540:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177232.078542:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078547:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177232.078548:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078549:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177232.078550:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078551:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177232.078552:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177232.078553:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177232.078555:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177232.078557:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177232.078559:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177232.078563:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592192:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177232.078566:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078567:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177232.078569:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb800 x1872930235592192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177232.078572:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177232.078573:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177232.078575:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177232.078576:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177232.078577:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177232.078578:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177232.078580:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177232.078581:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:2.0:1786177232.078582:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177232.078583:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177232.078584:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb800. 00000100:00000001:2.0:1786177232.078586:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177232.078587:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078588:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177232.078591:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177232.078592:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177232.078593:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177232.078594:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.588476:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177232.588485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177232.588488:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177232.588501:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177232.588504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177232.588507:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177232.588509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177232.588511:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177232.588513:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.972436:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177232.972441:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177232.972444:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177232.972446:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177232.972446:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177232.972448:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177232.972452:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177232.972452:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177232.972455:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177232.972455:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177232.972457:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.972459:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177232.972460:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177232.972462:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177232.972462:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177232.972465:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177232.972466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177232.972467:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177232.972468:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177232.972471:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177232.972472:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177232.972474:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177232.972476:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177232.972478:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177232.972479:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177232.972480:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177232.972482:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177232.972550:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177232.972555:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177232.972558:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177233.612578:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177233.612585:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177233.612588:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177233.612593:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177233.612598:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177233.612601:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177233.612604:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177233.612607:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177233.612611:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177233.996485:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177233.996488:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177233.996494:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177233.996494:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177233.996498:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177233.996503:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177233.996505:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177233.996507:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177233.996511:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177233.996514:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177233.996516:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177233.996518:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177233.996521:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177233.996526:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177233.996530:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177233.996582:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177233.996587:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177233.996591:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177234.636477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177234.636486:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177234.636489:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177234.636495:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177234.636500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177234.636504:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177234.636507:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177234.636511:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177234.636515:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177235.020449:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177235.020453:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177235.020456:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177235.020458:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177235.020459:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177235.020465:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177235.020467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177235.020469:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177235.020471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177235.020472:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177235.020476:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177235.020478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177235.020478:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177235.020483:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177235.020484:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177235.020547:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177235.020552:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177235.020556:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.0:1786177235.596494:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:1.0:1786177235.596501:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:1.0:1786177235.596506:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1786177235.596678:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1786177235.596684:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786177235.596693:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786177235.596697:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786177235.596699:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177235.660471:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177235.660479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177235.660482:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177235.660488:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177235.660492:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177235.660496:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177235.660499:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177235.660502:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177235.660506:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177235.980484:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306582784, at 4306582848 00000100:00000040:1.0:1786177235.980484:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306582784, at 4306582848 00000100:00000010:1.0:1786177235.980500:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c000. 00000100:00000010:0.0:1786177235.980503:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c300. 00000100:00000001:1.0:1786177236.044464:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177236.044470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177236.044478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177236.044479:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177236.044482:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177236.044486:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177236.044487:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177236.044489:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177236.044493:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177236.044494:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177236.044495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177236.044498:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177236.044501:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177236.044504:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177236.044505:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177236.044566:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177236.044571:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177236.044574:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177236.684458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177236.684466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177236.684468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177236.684479:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177236.684484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177236.684488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177236.684491:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177236.684494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177236.684496:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.068439:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177237.068444:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177237.068448:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.068449:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177237.068453:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177237.068455:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177237.068457:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177237.068461:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177237.068461:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177237.068463:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.068469:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177237.068474:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177237.068477:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177237.068480:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177237.068483:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177237.068530:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177237.068534:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177237.068538:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177237.196465:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177237.196474:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177237.196477:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177237.196480:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177237.196483:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177237.196489:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11911 00000400:00000200:1.0:1786177237.196496:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177237.196502:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177237.196505:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177237.196507:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196511:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177237.196514:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196517:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177237.196521:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177237.196523:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177237.196527:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177237.196530:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177237.196533:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177237.196535:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177237.196539:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196541:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177237.196543:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196545:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177237.196548:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177237.196552:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196555:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177237.196558:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177237.196562:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177237.196564:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.196566:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177237.196568:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177237.196576:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9f80. 00000020:00000040:1.0:1786177237.196579:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177237.196582:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177237.196584:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.196586:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177237.196588:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177237.196592:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177237.196597:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae400. 00000100:00000001:1.0:1786177237.196603:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177237.196610:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177237.196622:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177237.196626:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177237.196629:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4f3e5a2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177237.196649:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177237.196652:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177237.196656:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177237.196658:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177237.196660:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177237.196663:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177237.196665:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11911 00000400:00000200:1.0:1786177237.196670:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177237.196673:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177237.196676:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177237.196680:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196683:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177237.196685:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196687:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177237.196690:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177237.196692:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177237.196694:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177237.196696:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177237.196699:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177237.196699:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1786177237.196704:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177237.196704:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177237.196708:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177237.196709:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196712:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177237.196715:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196717:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:2.0:1786177237.196717:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177237.196722:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:1.0:1786177237.196727:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196732:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177237.196735:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177237.196737:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177237.196740:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177237.196741:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177237.196743:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177237.196745:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.196748:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:2.0:1786177237.196748:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177237.196753:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1786177237.196754:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1786177237.196758:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235592320:192.168.202.132@tcp:250:kworker.0 00000008:00000001:1.0:1786177237.196759:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.196763:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.196765:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1786177237.196767:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb480. 02000000:00000001:2.0:1786177237.196770:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:1.0:1786177237.196771:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177237.196774:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177237.196775:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177237.196777:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1786177237.196778:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.196781:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177237.196782:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177237.196784:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177237.196784:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177237.196789:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177237.196792:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac400. 00000400:00000200:2.0:1786177237.196796:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177237.196797:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177237.196801:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096688. 00000100:00100000:1.0:1786177237.196805:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1786177237.196808:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000001:1.0:1786177237.196815:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177237.196817:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177237.196817:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235592320, portal 25 00000100:00000001:2.0:1786177237.196823:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1786177237.196825:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b192a357] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177237.196826:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487990144 : -110371221561472 : ffff9b9e328b9f80) 00000100:00000040:2.0:1786177237.196835:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177237.196844:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.196845:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1786177237.196850:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1786177237.196850:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177237.196854:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28110. 00000100:00080000:1.0:1786177237.196859:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000200:2.0:1786177237.196859:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235592320, offset 0 00000100:00000001:1.0:1786177237.196863:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177237.196866:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1786177237.196867:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196873:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177237.196878:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177237.196879:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11911 00000400:00000200:1.0:1786177237.196885:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177237.196888:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177237.196892:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177237.196895:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177237.196896:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1786177237.196899:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196903:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177237.196903:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:1.0:1786177237.196907:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177237.196910:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1786177237.196911:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177237.196915:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000001:1.0:1786177237.196916:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177237.196919:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177237.196922:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:2.0:1786177237.196922:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426694:426694:256:4294967295] 192.168.202.132@tcp LPNI seq info [426656:426656:8:4294967295] 00000020:00000040:1.0:1786177237.196924:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177237.196929:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786177237.196933:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:1.0:1786177237.196934:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177237.196938:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196940:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177237.196961:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000040:1.0:1786177237.196965:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:2.0:1786177237.196970:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196971:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177237.196975:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000010:2.0:1786177237.196977:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d000. 00000100:00080000:1.0:1786177237.196979:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177237.196982:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177237.196984:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177237.196987:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1786177237.196987:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:1.0:1786177237.196991:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.196996:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:2.1:1786177237.196997:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000008:00100000:1.0:1786177237.196999:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177237.197002:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.197003:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177237.197006:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb100. 00000020:00000040:1.0:1786177237.197009:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000001:2.0:1786177237.197011:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177237.197012:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177237.197017:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177237.197017:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177237.197023:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1786177237.197023:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306587904 02000000:00000001:1.0:1786177237.197028:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177237.197030:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177237.197030:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290ce40. 00000100:00000001:2.0:1786177237.197038:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 02000000:00000010:1.0:1786177237.197039:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac800. 00000100:00000001:1.0:1786177237.197046:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.197046:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306587904 00000100:00000010:2.0:1786177237.197052:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c9c0. 00000100:00000001:2.0:1786177237.197057:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00100000:1.0:1786177237.197058:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177237.197060:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177237.197065:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.197070:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.197075:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177237.197076:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177237.197080:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.197082:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1786177237.197087:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f57195c8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177237.197087:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1786177237.197093:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177237.197097:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177237.197100:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177237.197105:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177237.197109:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1786177237.197109:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177237.197116:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177237.197118:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177237.197121:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177237.197127:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177237.197132:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11911 00000100:00000001:2.0:1786177237.197132:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1786177237.197136:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177237.197136:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177237.197142:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1786177237.197143:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 02000000:00000001:2.0:1786177237.197146:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1786177237.197147:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1786177237.197152:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177237.197153:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.197158:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1786177237.197161:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592448:192.168.202.132@tcp:38:kworker.0 00000100:00000040:1.0:1786177237.197163:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177237.197167:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1786177237.197168:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:2.0:1786177237.197170:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1786177237.197171:0:191280:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:2.0:1786177237.197173:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177237.197174:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177237.197178:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:2.0:1786177237.197178:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:1.0:1786177237.197183:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177237.197185:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 02000000:00000010:2.0:1786177237.197185:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177237.197189:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177237.197195:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:1.0:1786177237.197197:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000010:2.0:1786177237.197199:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096318. 00000100:00000040:1.0:1786177237.197202:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.197205:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1786177237.197206:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28908. 00000100:00000040:1.0:1786177237.197207:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.197211:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177237.197214:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000200:2.0:1786177237.197214:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235592448, portal 10 00000100:00080000:1.0:1786177237.197218:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177237.197222:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177237.197223:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.197226:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487995520 : -110371221556096 : ffff9b9e328bb480) 02000000:00000001:1.0:1786177237.197227:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177237.197232:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177237.197235:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177237.197239:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177237.197240:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177237.197244:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177237.197251:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.197255:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.197261:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:1.0:1786177237.197262:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328ba680. 00000100:00000040:2.0:1786177237.197267:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000020:00000040:1.0:1786177237.197268:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177237.197274:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:2.0:1786177237.197274:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c283b8. 02000000:00000001:1.0:1786177237.197276:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.197281:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:2.0:1786177237.197281:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235592448, offset 0 02000000:00000001:1.0:1786177237.197284:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177237.197288:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177237.197290:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 02000000:00000010:1.0:1786177237.197294:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae800. 00000400:00000200:2.0:1786177237.197299:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177237.197301:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177237.197306:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00100000:1.0:1786177237.197312:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786177237.197314:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177237.197322:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177237.197328:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1786177237.197329:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1786177237.197333:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.197335:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177237.197340:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426695:426695:255:4294967295] 192.168.202.132@tcp LPNI seq info [426657:426657:7:4294967295] 00000100:00000040:1.0:1786177237.197346:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000497a117c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177237.197350:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177237.197386:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177237.197399:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177237.197401:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1786177237.197405:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6dd00. 00000100:00080000:1.0:1786177237.197406:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177237.197412:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000200:2.0:1786177237.197414:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177237.197415:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177237.197419:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1786177237.197424:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177237.197426:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786177237.197431:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177237.197433:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11911 00000100:00000040:2.0:1786177237.197438:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177237.197441:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177237.197446:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177237.197450:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177237.197452:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306584068 00000100:00080000:1.0:1786177237.197455:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177237.197458:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306587904 00000100:00000040:1.0:1786177237.197462:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177237.197463:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306584068 00000100:00000001:1.0:1786177237.197468:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177237.197469:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306587904 00000100:00000040:1.0:1786177237.197474:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177237.197476:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177237.197480:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177237.197482:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177237.197487:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177237.197489:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177237.197491:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000001:2.0:1786177237.197494:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1786177237.197495:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177237.197499:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:2.0:1786177237.197500:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1786177237.197503:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177237.197504:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000001:1.0:1786177237.197508:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177237.197509:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177237.197513:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1786177237.197517:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177237.197517:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177237.197522:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177237.197526:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177237.197527:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177237.197532:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177237.197538:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:1.0:1786177237.197544:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177237.197547:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177237.197549:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.197550:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177237.197554:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177237.197556:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177237.197560:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177237.197560:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177237.197567:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177237.197567:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.197571:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:2.0:1786177237.197576:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592576:192.168.202.132@tcp:38:kworker.0 00000008:00100000:1.0:1786177237.197577:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177237.197584:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.197586:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1786177237.197587:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177237.197590:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:1.0:1786177237.197593:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9c00. 02000000:00000001:2.0:1786177237.197594:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177237.197598:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177237.197599:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1786177237.197605:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000010:2.0:1786177237.197606:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:1.0:1786177237.197607:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.197613:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177237.197613:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177237.197616:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177237.197620:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177237.197621:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177237.197627:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e10096d68. 02000000:00000010:1.0:1786177237.197629:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac000. 00000400:00000010:2.0:1786177237.197632:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28000. 00000100:00000001:1.0:1786177237.197634:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177237.197638:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235592576, portal 10 00000100:00100000:1.0:1786177237.197643:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177237.197644:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177237.197647:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487994624 : -110371221556992 : ffff9b9e328bb100) 00000100:00000001:1.0:1786177237.197656:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1786177237.197658:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177237.197661:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177237.197671:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3ea07ed] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177237.197672:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177237.197679:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177237.197684:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28aa0. 00000100:00000001:1.0:1786177237.197689:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177237.197689:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235592576, offset 0 00000100:00000040:1.0:1786177237.197698:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11921) 00000400:00000200:2.0:1786177237.197699:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177237.197706:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177237.197710:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177237.197714:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177237.197719:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177237.197722:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177237.197725:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177237.197728:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426696:426696:254:4294967295] 192.168.202.132@tcp LPNI seq info [426658:426658:6:4294967295] 00000400:00000200:2.0:1786177237.197732:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177237.197738:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177237.197743:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177237.197747:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d400. 00000800:00000200:2.0:1786177237.197752:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177237.197756:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177237.197759:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177237.197762:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177237.197766:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177237.197768:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306584068 00000100:00000001:2.0:1786177237.197770:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306587904 00000100:00000001:2.0:1786177237.197772:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306584068 00000100:00000001:2.0:1786177237.197775:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306587904 00000100:00000001:2.0:1786177237.197777:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177237.197779:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.197781:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177237.197785:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177237.197793:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177237.197794:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177237.197796:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177237.197798:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177237.197800:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177237.197804:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592704:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177237.197807:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177237.197809:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177237.197811:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177237.197813:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177237.197816:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177237.197819:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177237.197822:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177237.197825:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100969a0. 00000400:00000010:2.0:1786177237.197829:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28330. 00000100:00000200:2.0:1786177237.197832:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235592704, portal 4 00000100:00000001:2.0:1786177237.197834:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177237.197835:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487991936 : -110371221559680 : ffff9b9e328ba680) 00000100:00000040:2.0:1786177237.197842:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177237.197849:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177237.197851:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177237.197854:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28880. 00000100:00000200:2.0:1786177237.197856:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235592704, offset 0 00000400:00000200:2.0:1786177237.197861:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177237.197865:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177237.197869:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177237.197872:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177237.197875:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177237.197877:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177237.197879:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177237.197882:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426697:426697:253:4294967295] 192.168.202.132@tcp LPNI seq info [426659:426659:5:4294967295] 00000400:00000200:2.0:1786177237.197886:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177237.197893:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177237.197899:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177237.197903:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d700. 00000800:00000200:2.0:1786177237.197907:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177237.197911:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177237.197914:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177237.197918:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177237.197921:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177237.197923:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306584068 00000100:00000001:2.0:1786177237.197925:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306587904 00000100:00000001:2.0:1786177237.197928:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306584068 00000100:00000001:2.0:1786177237.197930:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306587904 00000100:00000001:2.0:1786177237.197932:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177237.197935:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.197937:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.197940:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177237.197943:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177237.197957:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1786177237.197960:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177237.197961:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177237.197963:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177237.197968:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177237.197976:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177237.197977:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177237.197979:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177237.197981:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177237.197983:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177237.197986:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592832:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177237.197990:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177237.197992:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177237.197994:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177237.197995:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177237.197999:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177237.198002:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177237.198005:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177237.198009:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177237.198013:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177237.198016:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235592832, portal 4 00000100:00000001:2.0:1786177237.198019:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177237.198020:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487989248 : -110371221562368 : ffff9b9e328b9c00) 00000100:00000040:2.0:1786177237.198027:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177237.198034:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177237.198037:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177237.198039:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28dd0. 00000100:00000200:2.0:1786177237.198042:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235592832, offset 0 00000400:00000200:2.0:1786177237.198045:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177237.198048:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177237.198051:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177237.198055:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177237.198058:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177237.198060:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177237.198062:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177237.198065:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426698:426698:252:4294967295] 192.168.202.132@tcp LPNI seq info [426660:426660:4:4294967295] 00000400:00000200:2.0:1786177237.198070:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177237.198076:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177237.198081:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177237.198084:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d200. 00000800:00000200:2.0:1786177237.198088:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177237.198092:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177237.198095:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177237.198098:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177237.198101:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177237.198103:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306584070 00000100:00000001:2.0:1786177237.198106:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306587904 00000100:00000001:2.0:1786177237.198109:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306584070 00000100:00000001:2.0:1786177237.198111:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306587904 00000100:00000001:2.0:1786177237.198113:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177237.198115:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.198118:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.198120:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177237.198123:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177237.198125:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1786177237.198127:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177237.198128:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177237.198130:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177237.198132:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177237.708457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177237.708466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177237.708468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177237.708483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177237.708490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177237.708494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177237.708497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177237.708499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177237.708503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177238.092454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177238.092460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177238.092461:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177238.092462:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177238.092467:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177238.092469:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177238.092473:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177238.092475:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177238.092477:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177238.092480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177238.092481:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177238.092484:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177238.092487:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177238.092488:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177238.092492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177238.092551:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177238.092554:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177238.092558:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177238.732472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177238.732491:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177238.732494:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177238.732500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177238.732504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177238.732508:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177238.732511:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177238.732515:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177238.732519:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177239.116459:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177239.116466:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177239.116467:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177239.116471:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177239.116475:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177239.116479:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177239.116482:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177239.116485:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177239.116488:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177239.116660:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177239.116663:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177239.116666:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177239.116667:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177239.116669:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177239.116670:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177239.116725:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177239.116729:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177239.116732:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177239.756465:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177239.756472:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177239.756474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177239.756479:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177239.756483:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177239.756486:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177239.756489:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177239.756491:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177239.756493:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177240.140489:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177240.140495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177240.140499:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177240.140500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177240.140502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177240.140507:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177240.140507:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177240.140509:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177240.140512:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177240.140518:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177240.140522:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177240.140527:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177240.140536:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177240.140540:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177240.140544:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177240.140566:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177240.140570:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177240.140573:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1786177240.268568:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:3.0:1786177240.268611:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1786177240.268623:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177240.268627:0:2381:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:3.0:1786177240.268632:0:2381:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177240.268636:0:2381:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1786177240.268641:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786177240.268645:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d000. 00000400:00000200:3.0:1786177240.268650:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786177240.268655:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786177240.268662:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28110 00000400:00000010:3.0:1786177240.268664:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28110. 00000100:00000001:3.0:1786177240.268670:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786177240.268679:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177240.268704:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786177240.268709:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177240.268717:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177240.268719:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786177240.268729:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786177240.268732:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6dd00. 00000400:00000200:3.0:1786177240.268734:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786177240.268737:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786177240.268740:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c283b8 00000400:00000010:3.0:1786177240.268742:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c283b8. 00000100:00000001:3.0:1786177240.268745:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786177240.268749:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177240.268757:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786177240.268760:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177240.268767:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177240.268769:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786177240.268773:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786177240.268776:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d400. 00000400:00000200:3.0:1786177240.268779:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786177240.268782:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786177240.268785:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28aa0 00000400:00000010:3.0:1786177240.268788:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28aa0. 00000100:00000001:3.0:1786177240.268790:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1786177240.268794:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177240.268801:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786177240.268805:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177240.268811:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177240.268813:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786177240.268817:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786177240.268820:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d700. 00000400:00000200:3.0:1786177240.268824:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786177240.268828:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786177240.268831:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28880 00000400:00000010:3.0:1786177240.268833:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28880. 00000100:00000001:3.0:1786177240.268836:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1786177240.268839:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:3.0:1786177240.268842:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.268847:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177240.268854:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1786177240.268856:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1786177240.268863:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177240.268868:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177237/real 1786177240] req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177240.268873:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177240.268875:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1786177240.268882:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:3.0:1786177240.268887:0:2381:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d200. 00000400:00000200:3.0:1786177240.268890:0:2381:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1786177240.268901:0:2381:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1786177240.268907:0:2381:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28dd0 00000400:00000010:3.0:1786177240.268910:0:2381:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28dd0. 00000400:00000001:2.0:1786177240.268911:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1786177240.268914:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:2.0:1786177240.268925:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177240.268932:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096d68. 00000100:00000200:3.0:1786177240.268933:0:2381:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177240.268939:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28000 00000400:00000010:2.0:1786177240.268945:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28000. 00000100:00000001:3.0:1786177240.268952:0:2381:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1786177240.268953:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1786177240.268959:0:2381:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177262 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177240.268963:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177240.268979:0:2381:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177240.268983:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:2.0:1786177240.268989:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269000:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177240.269002:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269006:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177240.269009:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177240.269013:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269021:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177240.269027:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269036:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177240.269040:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269041:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177240.269042:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269045:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177240.269049:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177240.269053:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177240.269059:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177240.269062:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177240.269067:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177240.269075:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592576:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177240.269080:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177240.269084:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177237/real 1786177240] req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177240.269092:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177240.269093:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177240.269097:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100969a0. 00000400:00000200:2.0:1786177240.269099:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28330 00000400:00000010:2.0:1786177240.269101:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28330. 00000100:00000001:2.0:1786177240.269104:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177240.269107:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177240.269117:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269123:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177240.269125:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269126:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177240.269128:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177240.269132:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269139:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177240.269143:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269150:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177240.269153:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269154:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177240.269156:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269158:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177240.269160:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177240.269163:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177240.269167:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177240.269170:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177240.269173:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177240.269182:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592704:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177240.269186:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177240.269190:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177237/real 1786177240] req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177240.269198:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177240.269200:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177240.269202:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096318. 00000400:00000200:2.0:1786177240.269205:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28908 00000400:00000010:2.0:1786177240.269207:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28908. 00000100:00000001:2.0:1786177240.269209:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177240.269225:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177240.269234:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269241:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177240.269243:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269245:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177240.269247:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177240.269251:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269258:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177240.269262:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269269:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177240.269271:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269273:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177240.269274:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269276:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177240.269279:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177240.269281:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177240.269286:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177240.269288:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177240.269292:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177240.269300:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592448:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177240.269304:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177240.269307:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177237/real 1786177240] req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177240.269315:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177240.269317:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177240.269319:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e10096688. 00000400:00000200:2.0:1786177240.269322:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177240.269324:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177240.269327:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177240.269330:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177240.269340:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269347:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177240.269348:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269350:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177240.269352:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177240.269384:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269393:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177240.269398:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269405:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177240.269408:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269409:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177240.269410:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269412:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177240.269414:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177240.269417:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177240.269421:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177240.269423:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177240.269427:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177240.269435:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235592320:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177240.269440:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177240.269442:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177240.269446:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb100 x1872930235592576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269453:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177240.269456:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177240.269459:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177240.269464:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177240.269467:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177240.269469:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177240.269472:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177240.269475:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:2.0:1786177240.269478:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177240.269480:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177240.269482:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb100. 00000100:00000001:2.0:1786177240.269486:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177240.269494:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269497:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177240.269501:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328ba680 x1872930235592704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269513:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177240.269515:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177240.269516:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177240.269519:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177240.269521:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177240.269522:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177240.269524:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177240.269526:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:2.0:1786177240.269529:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177240.269530:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177240.269531:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328ba680. 00000100:00000001:2.0:1786177240.269535:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177240.269536:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269538:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177240.269542:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb480 x1872930235592448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269550:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177240.269552:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177240.269554:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177240.269556:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177240.269557:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177240.269559:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177240.269561:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177240.269563:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:2.0:1786177240.269565:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177240.269566:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177240.269567:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb480. 00000100:00000001:2.0:1786177240.269570:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177240.269571:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269573:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177240.269576:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9f80 x1872930235592320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269583:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177240.269584:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177240.269587:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177240.269589:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177240.269590:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177240.269592:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177240.269593:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177240.269595:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:2.0:1786177240.269597:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177240.269599:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177240.269600:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9f80. 00000100:00000001:2.0:1786177240.269603:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177240.269604:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269607:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177240.269609:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177240.269611:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177240.269612:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177240.269617:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177237/real 1786177240] req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177240.269625:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177240.269627:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177240.269629:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177240.269632:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177240.269634:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177240.269636:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177240.269640:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177240.269648:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269655:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177240.269656:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269658:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177240.269659:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177240.269663:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269669:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177240.269674:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269681:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177240.269684:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269685:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177240.269687:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269689:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177240.269691:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177240.269693:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177240.269697:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177240.269699:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177240.269704:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177240.269713:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235592832:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177240.269718:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269720:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177240.269723:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9c00 x1872930235592832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177262 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177240.269729:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177240.269731:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177240.269733:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177240.269736:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177240.269737:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177240.269738:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177240.269741:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177240.269742:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:2.0:1786177240.269744:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177240.269746:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177240.269747:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9c00. 00000100:00000001:2.0:1786177240.269750:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177240.269751:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269753:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177240.269761:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177240.269763:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177240.269766:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177240.269767:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177240.780479:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177240.780487:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177240.780490:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177240.780495:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177240.780500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177240.780504:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177240.780508:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177240.780510:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177240.780513:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177241.100464:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306587904, at 4306587968 00000100:00000010:1.0:1786177241.100484:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c9c0. 00000100:00000040:0.0:1786177241.100575:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306587904, at 4306587968 00000100:00000010:0.0:1786177241.100581:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290ce40. 00000100:00000001:1.0:1786177241.164430:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177241.164437:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177241.164440:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177241.164444:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177241.164448:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177241.164451:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177241.164453:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177241.164455:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177241.164458:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177241.164460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177241.164464:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177241.164470:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177241.164473:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177241.164482:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177241.164487:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177241.164490:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177241.164493:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177241.164495:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177241.164498:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177241.164502:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177241.164504:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177241.164558:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177241.164561:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177241.164565:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177241.804474:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177241.804483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177241.804486:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177241.804492:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177241.804497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177241.804501:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177241.804505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177241.804509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177241.804512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177242.188478:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177242.188483:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177242.188487:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177242.188487:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177242.188491:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177242.188496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177242.188498:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177242.188500:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177242.188504:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177242.188507:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177242.188510:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177242.188510:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177242.188517:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177242.188521:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177242.188524:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177242.188572:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177242.188576:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177242.188579:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177242.316490:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177242.316499:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177242.316501:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177242.316504:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177242.316507:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177242.316513:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11916 00000400:00000200:3.0:1786177242.316520:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177242.316525:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177242.316528:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177242.316530:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316533:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177242.316535:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316536:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177242.316540:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177242.316541:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177242.316543:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177242.316546:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177242.316547:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177242.316549:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177242.316551:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316553:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177242.316555:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316556:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177242.316558:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177242.316561:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316562:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177242.316564:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177242.316567:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177242.316569:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.316570:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177242.316572:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177242.316577:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786177242.316580:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177242.316583:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177242.316584:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.316586:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177242.316589:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177242.316592:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177242.316597:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786177242.316603:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177242.316615:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177242.316623:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177242.316627:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177242.316631:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177242.316659:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177242.316662:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177242.316666:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177242.316668:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177242.316669:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177242.316672:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177242.316675:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11916 00000400:00000200:3.0:1786177242.316678:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177242.316682:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177242.316684:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177242.316687:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316689:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177242.316691:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316693:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177242.316696:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177242.316697:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177242.316698:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177242.316700:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177242.316702:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177242.316704:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177242.316707:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316709:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177242.316711:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316713:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177242.316716:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177242.316719:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316721:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177242.316723:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177242.316729:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177242.316731:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.316733:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177242.316737:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177242.316740:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.316742:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177242.316745:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786177242.316747:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177242.316750:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177242.316752:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.316753:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177242.316755:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177242.316757:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177242.316761:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786177242.316765:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177242.316771:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177242.316778:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177242.316780:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177242.316783:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177242.316791:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177242.316794:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177242.316797:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177242.316799:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177242.316800:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177242.316803:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177242.316806:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11916 00000400:00000200:3.0:1786177242.316810:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177242.316813:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177242.316815:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177242.316818:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316820:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177242.316822:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316824:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177242.316826:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177242.316827:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177242.316828:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177242.316830:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177242.316832:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177242.316835:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177242.316837:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316839:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177242.316841:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316843:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177242.316845:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177242.316847:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316850:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177242.316851:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177242.316854:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177242.316856:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.316858:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177242.316860:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177242.316862:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.316863:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177242.316866:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786177242.316869:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177242.316871:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177242.316872:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.316874:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177242.316876:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177242.316877:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177242.316881:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786177242.316884:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177242.316888:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177242.316895:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177242.316933:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177242.316938:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177242.316945:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177242.316948:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177242.316952:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177242.316954:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177242.316955:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177242.316958:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177242.316961:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11916 00000400:00000200:3.0:1786177242.316965:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177242.316968:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177242.316970:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177242.316974:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316976:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177242.316978:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316980:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177242.316982:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177242.316983:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177242.316984:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177242.316987:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177242.316989:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177242.316991:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177242.316994:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.316996:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177242.316998:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.317000:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177242.317003:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177242.317005:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177242.317008:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177242.317010:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177242.317013:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177242.317015:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.317018:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177242.317020:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177242.317023:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.317025:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177242.317028:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786177242.317031:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177242.317033:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177242.317035:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.317037:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177242.317038:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177242.317040:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177242.317043:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786177242.317047:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177242.317052:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177242.317060:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177242.317062:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177242.317066:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177242.317074:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177242.317079:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177242.317082:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177242.317084:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177242.317086:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177242.317088:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177242.317091:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11916 00000400:00000200:3.0:1786177242.317094:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177242.317097:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177242.317099:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177242.317102:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.317104:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177242.317106:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.317109:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177242.317111:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177242.317112:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177242.317114:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177242.317115:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177242.317118:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177242.317120:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177242.317123:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.317125:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177242.317127:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177242.317129:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177242.317131:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177242.317134:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177242.317137:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177242.317138:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177242.317141:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177242.317143:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.317145:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177242.317147:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177242.317149:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.317150:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177242.317154:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786177242.317156:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177242.317158:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177242.317159:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177242.317161:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177242.317162:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177242.317164:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177242.317167:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177242.317171:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177242.317176:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177242.317189:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177242.317191:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177242.317195:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177242.317203:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177242.317288:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11926) 00000100:00000001:2.0:1786177242.317390:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177242.317396:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177242.317398:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177242.317405:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.317414:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177242.317417:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177242.317421:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177242.317424:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177242.317427:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177242.317430:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235592960:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177242.317434:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177242.317439:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177242.317442:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177242.317445:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177242.317450:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177242.317453:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177242.317459:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177242.317462:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177242.317470:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177242.317478:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235592960, portal 25 00000100:00000001:2.0:1786177242.317481:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177242.317482:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280811776 : -110371428739840 : ffff9b9e26325500) 00000100:00000040:2.0:1786177242.317488:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.317495:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177242.317499:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177242.317501:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177242.317505:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235592960, offset 0 00000400:00000200:2.0:1786177242.317512:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177242.317519:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177242.317527:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177242.317534:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177242.317540:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177242.317544:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177242.317547:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177242.317552:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426699:426699:256:4294967295] 192.168.202.132@tcp LPNI seq info [426661:426661:8:4294967295] 00000400:00000200:2.0:1786177242.317557:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177242.317569:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177242.317576:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177242.317583:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177242.317591:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177242.317596:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786177242.317603:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177242.317606:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1786177242.317611:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177242.317614:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177242.317618:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177242.317622:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177242.317628:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177242.317630:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177242.317634:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177242.317638:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.317649:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177242.317652:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.317660:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177242.317662:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177242.317668:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177242.317673:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177242.317675:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306593025 00000100:00000010:2.0:1786177242.317681:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cb40. 00000100:00000001:2.0:1786177242.317687:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177242.317693:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306593025 00000100:00000010:2.0:1786177242.317696:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c660. 00000100:00000001:2.0:1786177242.317699:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177242.317702:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177242.317704:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177242.317707:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177242.317711:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177242/real 1786177242] req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177242.317720:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177242.317723:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177242.317725:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177242.317727:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177242.317729:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177242.317732:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177242.317735:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177242.317743:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.317750:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177242.317751:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.317754:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177242.317756:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177242.317760:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.317769:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177242.317774:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.317781:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177242.317784:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.317785:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177242.317786:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.317789:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177242.317791:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177242.317794:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177242.317798:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177242.317801:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177242.317806:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177242.317814:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235592960:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177242.317819:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177242.317822:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.317828:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177242.317830:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177242.317832:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177242.317834:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177242.317837:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177242.317840:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593088:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177242.317843:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177242.317845:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177242.317847:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177242.317849:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177242.317853:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177242.317855:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177242.317859:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177242.317861:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177242.317865:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177242.317869:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235593088, portal 10 00000100:00000001:2.0:1786177242.317871:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177242.317872:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280812672 : -110371428738944 : ffff9b9e26325880) 00000100:00000040:2.0:1786177242.317879:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.317886:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177242.317889:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177242.317891:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177242.317893:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235593088, offset 0 00000400:00000200:2.0:1786177242.317915:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177242.317919:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177242.317923:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177242.317927:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177242.317930:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177242.317932:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177242.317935:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177242.317938:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426700:426700:256:4294967295] 192.168.202.132@tcp LPNI seq info [426662:426662:8:4294967295] 00000400:00000200:2.0:1786177242.317942:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177242.317950:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177242.317954:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177242.317958:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177242.317963:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177242.317966:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786177242.317971:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177242.317977:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1786177242.317980:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177242.317982:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177242.317985:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177242.317989:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177242.317992:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177242.317994:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177242.317996:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177242.318000:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318007:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177242.318011:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318018:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177242.318019:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177242.318023:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177242.318028:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177242.318031:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177242.318033:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306589188 00000100:00000001:2.0:1786177242.318036:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306593025 00000100:00000001:2.0:1786177242.318038:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306589188 00000100:00000001:2.0:1786177242.318040:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306593025 00000100:00000001:2.0:1786177242.318043:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177242.318045:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177242.318047:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177242.318051:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177242/real 1786177242] req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177242.318058:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177242.318059:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177242.318061:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177242.318064:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177242.318066:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177242.318068:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177242.318072:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177242.318079:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318085:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177242.318086:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.318088:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177242.318089:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177242.318093:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318099:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177242.318102:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318108:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177242.318111:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.318112:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177242.318113:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.318116:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177242.318119:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177242.318121:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177242.318124:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177242.318127:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177242.318131:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177242.318138:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593088:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177242.318142:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177242.318145:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318157:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177242.318158:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177242.318160:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177242.318162:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177242.318164:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177242.318167:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593216:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177242.318171:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177242.318173:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177242.318175:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177242.318177:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177242.318180:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177242.318183:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177242.318186:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177242.318190:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177242.318193:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177242.318196:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235593216, portal 10 00000100:00000001:2.0:1786177242.318198:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177242.318199:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280821632 : -110371428729984 : ffff9b9e26327b80) 00000100:00000040:2.0:1786177242.318205:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318212:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177242.318214:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177242.318217:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177242.318220:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235593216, offset 0 00000400:00000200:2.0:1786177242.318223:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177242.318227:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177242.318231:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177242.318234:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177242.318237:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177242.318239:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177242.318242:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177242.318245:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426701:426701:256:4294967295] 192.168.202.132@tcp LPNI seq info [426663:426663:8:4294967295] 00000400:00000200:2.0:1786177242.318249:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177242.318255:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177242.318259:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177242.318263:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177242.318267:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177242.318270:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786177242.318274:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177242.318277:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1786177242.318281:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177242.318283:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177242.318286:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177242.318290:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177242.318293:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177242.318295:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177242.318298:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177242.318302:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318309:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177242.318312:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318318:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177242.318320:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177242.318322:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177242.318326:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177242.318329:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177242.318331:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306589188 00000100:00000001:2.0:1786177242.318334:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306593025 00000100:00000001:2.0:1786177242.318336:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306589188 00000100:00000001:2.0:1786177242.318338:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306593025 00000100:00000001:2.0:1786177242.318340:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177242.318343:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177242.318344:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177242.318348:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177242/real 1786177242] req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177242.318390:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177242.318392:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177242.318394:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177242.318398:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177242.318399:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177242.318402:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177242.318406:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177242.318415:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318422:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177242.318423:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.318425:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177242.318427:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177242.318431:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318438:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177242.318443:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318450:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177242.318452:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.318454:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177242.318455:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.318457:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177242.318459:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177242.318462:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177242.318465:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177242.318467:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177242.318472:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177242.318480:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593216:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177242.318484:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177242.318489:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318495:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177242.318496:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177242.318499:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177242.318501:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177242.318504:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177242.318507:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593344:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177242.318511:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177242.318514:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177242.318516:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177242.318518:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177242.318522:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177242.318525:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177242.318530:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177242.318534:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177242.318537:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177242.318540:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235593344, portal 4 00000100:00000001:2.0:1786177242.318543:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177242.318544:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280815360 : -110371428736256 : ffff9b9e26326300) 00000100:00000040:2.0:1786177242.318552:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318558:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177242.318562:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177242.318564:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177242.318567:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235593344, offset 0 00000400:00000200:2.0:1786177242.318571:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177242.318575:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177242.318578:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177242.318582:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177242.318584:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177242.318587:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177242.318589:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177242.318592:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426702:426702:256:4294967295] 192.168.202.132@tcp LPNI seq info [426664:426664:8:4294967295] 00000400:00000200:2.0:1786177242.318597:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177242.318606:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177242.318612:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177242.318616:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177242.318620:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177242.318623:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786177242.318628:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177242.318631:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1786177242.318637:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177242.318639:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177242.318643:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177242.318646:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177242.318649:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177242.318651:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177242.318654:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177242.318658:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318667:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177242.318670:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318678:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177242.318680:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177242.318684:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177242.318687:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177242.318690:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177242.318693:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306589189 00000100:00000001:2.0:1786177242.318695:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306593025 00000100:00000001:2.0:1786177242.318698:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306589189 00000100:00000001:2.0:1786177242.318700:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306593025 00000100:00000001:2.0:1786177242.318703:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177242.318705:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177242.318707:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177242.318711:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177242/real 1786177242] req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177242.318719:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177242.318720:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177242.318722:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177242.318724:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177242.318726:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177242.318728:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177242.318731:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177242.318739:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318745:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177242.318747:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.318749:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177242.318750:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177242.318753:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318759:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177242.318763:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318769:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177242.318771:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.318773:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177242.318774:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.318776:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177242.318777:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177242.318786:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177242.318789:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177242.318791:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177242.318795:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177242.318802:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593344:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177242.318807:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177242.318810:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318816:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177242.318818:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177242.318820:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177242.318822:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177242.318824:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177242.318827:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593472:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177242.318830:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177242.318831:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177242.318833:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177242.318835:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177242.318838:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177242.318840:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177242.318843:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177242.318846:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177242.318849:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177242.318852:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235593472, portal 4 00000100:00000001:2.0:1786177242.318854:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177242.318856:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280813568 : -110371428738048 : ffff9b9e26325c00) 00000100:00000040:2.0:1786177242.318862:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318869:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177242.318871:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177242.318874:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177242.318876:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235593472, offset 0 00000400:00000200:2.0:1786177242.318880:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177242.318884:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177242.318888:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177242.318892:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177242.318894:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177242.318912:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177242.318914:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177242.318918:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426703:426703:256:4294967295] 192.168.202.132@tcp LPNI seq info [426665:426665:8:4294967295] 00000400:00000200:2.0:1786177242.318923:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177242.318929:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177242.318934:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177242.318938:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177242.318942:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177242.318945:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1786177242.318949:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177242.318952:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1786177242.318956:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177242.318959:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177242.318962:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177242.318965:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177242.318968:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177242.318970:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177242.318973:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177242.318976:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318983:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177242.318986:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177267 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.318992:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177242.318994:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177242.318996:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177242.319000:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177242.319003:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177242.319005:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306589191 00000100:00000001:2.0:1786177242.319007:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306593025 00000100:00000001:2.0:1786177242.319010:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306589191 00000100:00000001:2.0:1786177242.319012:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306593025 00000100:00000001:2.0:1786177242.319014:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177242.319016:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177242.319018:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177242.319022:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177242/real 1786177242] req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177242.319030:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177242.319031:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177242.319033:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177242.319035:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177242.319037:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177242.319039:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177242.319042:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177242.319051:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.319064:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177242.319065:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.319067:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177242.319069:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177242.319072:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.319080:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177242.319084:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.319091:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177242.319094:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.319095:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177242.319097:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.319099:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177242.319100:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177242.319103:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177242.319106:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177242.319109:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177242.319113:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177242.319121:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593472:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177242.319126:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.319128:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177242.319132:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325500 x1872930235592960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.319140:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177242.319142:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177242.319147:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177242.319150:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177242.319152:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177242.319154:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177242.319156:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177242.319159:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 02000000:00000001:2.0:1786177242.319162:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177242.319165:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177242.319167:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325500. 00000100:00000001:2.0:1786177242.319170:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177242.319171:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.319173:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177242.319177:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325880 x1872930235593088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.319184:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177242.319186:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177242.319187:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177242.319190:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177242.319192:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177242.319193:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177242.319195:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177242.319196:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:2.0:1786177242.319199:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177242.319200:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177242.319201:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325880. 00000100:00000001:2.0:1786177242.319204:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177242.319205:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.319207:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177242.319210:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327b80 x1872930235593216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.319218:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177242.319219:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177242.319221:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177242.319223:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177242.319224:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177242.319226:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177242.319228:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177242.319230:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:2.0:1786177242.319232:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177242.319233:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177242.319236:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327b80. 00000100:00000001:2.0:1786177242.319238:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177242.319239:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.319248:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177242.319252:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326300 x1872930235593344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.319259:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177242.319260:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177242.319262:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177242.319264:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177242.319266:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177242.319267:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177242.319269:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177242.319270:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:2.0:1786177242.319271:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177242.319272:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177242.319273:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326300. 00000100:00000001:2.0:1786177242.319275:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177242.319276:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.319277:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177242.319279:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325c00 x1872930235593472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177267 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177242.319284:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177242.319284:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177242.319286:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177242.319287:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177242.319287:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177242.319288:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177242.319289:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177242.319290:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:2.0:1786177242.319291:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177242.319293:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177242.319294:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325c00. 00000100:00000001:2.0:1786177242.319295:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177242.319296:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.319297:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177242.319303:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177242.319305:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177242.319306:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177242.319308:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177242.828453:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177242.828460:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177242.828461:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177242.828466:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177242.828470:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177242.828473:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177242.828475:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177242.828477:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177242.828480:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177243.212477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177243.212480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177243.212485:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177243.212487:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177243.212490:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177243.212495:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177243.212496:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177243.212497:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177243.212500:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177243.212501:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177243.212506:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177243.212508:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177243.212510:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177243.212513:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177243.212514:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177243.212515:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177243.212518:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177243.212519:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177243.212519:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177243.212522:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177243.212524:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177243.212524:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177243.212528:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177243.212528:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177243.212529:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177243.212532:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177243.212533:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177243.212604:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177243.212609:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177243.212612:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177243.852479:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177243.852489:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177243.852492:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177243.852499:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177243.852504:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177243.852508:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177243.852511:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177243.852514:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177243.852518:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177244.236490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177244.236492:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177244.236499:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177244.236502:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177244.236507:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177244.236507:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177244.236510:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177244.236515:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177244.236515:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177244.236517:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177244.236524:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177244.236529:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177244.236532:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177244.236535:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177244.236539:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177244.236591:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177244.236596:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177244.236600:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177244.876444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177244.876451:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177244.876453:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177244.876459:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177244.876462:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177244.876465:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177244.876467:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177244.876470:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177244.876472:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177245.260463:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177245.260467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177245.260470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177245.260471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177245.260474:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177245.260477:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177245.260479:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177245.260480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177245.260484:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177245.260485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177245.260488:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177245.260489:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177245.260493:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177245.260497:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177245.260499:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177245.260554:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177245.260559:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177245.260562:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177245.900472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177245.900482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177245.900485:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177245.900492:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177245.900497:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177245.900502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177245.900505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177245.900509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177245.900513:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177246.220427:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306593025, at 4306593088 00000100:00000010:0.0:1786177246.220441:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cb40. 00000100:00000040:1.0:1786177246.220447:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306593025, at 4306593088 00000100:00000010:1.0:1786177246.220458:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c660. 00000100:00000001:1.0:1786177246.284457:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177246.284464:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177246.284466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177246.284466:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177246.284472:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177246.284475:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177246.284479:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177246.284481:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177246.284484:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177246.284489:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177246.284490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177246.284494:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177246.284495:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177246.284500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177246.284504:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177246.284587:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177246.284592:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177246.284595:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177246.924478:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177246.924495:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177246.924498:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177246.924504:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177246.924508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177246.924512:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177246.924514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177246.924517:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177246.924521:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177247.308464:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177247.308468:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177247.308471:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177247.308472:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177247.308474:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177247.308479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177247.308479:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177247.308481:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177247.308484:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177247.308486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177247.308488:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177247.308489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177247.308493:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177247.308494:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177247.308496:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177247.308542:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177247.308545:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177247.308548:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177247.436467:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177247.436475:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177247.436478:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177247.436481:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177247.436485:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177247.436490:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11921 00000400:00000200:3.0:1786177247.436497:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177247.436503:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177247.436505:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177247.436508:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436510:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177247.436512:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436514:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177247.436517:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177247.436519:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177247.436521:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177247.436523:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177247.436525:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177247.436526:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177247.436528:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436530:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177247.436531:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436533:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177247.436536:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177247.436538:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436540:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177247.436542:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177247.436545:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177247.436547:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177247.436548:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177247.436550:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177247.436557:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786177247.436560:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177247.436563:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177247.436564:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177247.436566:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177247.436568:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177247.436572:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177247.436578:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786177247.436584:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177247.436591:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177247.436598:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177247.436602:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177247.436605:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177247.436623:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177247.436626:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177247.436630:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177247.436631:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177247.436633:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177247.436635:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177247.436638:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11921 00000400:00000200:3.0:1786177247.436642:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177247.436646:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177247.436648:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177247.436651:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436653:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177247.436655:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436657:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177247.436660:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177247.436661:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177247.436662:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177247.436665:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177247.436667:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177247.436669:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177247.436670:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1786177247.436673:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177247.436674:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177247.436674:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177247.436677:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1786177247.436677:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436679:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177247.436681:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177247.436684:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000040:2.0:1786177247.436685:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177247.436687:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177247.436689:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177247.436693:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1786177247.436694:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177247.436696:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177247.436696:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177247.436698:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:2.0:1786177247.436699:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177247.436702:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1786177247.436702:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1786177247.436705:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1786177247.436705:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177247.436708:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177247.436709:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235593600:192.168.202.132@tcp:250:kworker.0 00000100:00000010:3.0:1786177247.436711:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000100:00000001:2.0:1786177247.436713:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1786177247.436713:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177247.436716:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177247.436717:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.436718:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1786177247.436719:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177247.436720:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177247.436721:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177247.436723:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1786177247.436723:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177247.436726:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 02000000:00000010:2.0:1786177247.436728:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:3.0:1786177247.436729:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.436730:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177247.436734:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786177247.436736:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177247.436740:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000100:00000001:3.0:1786177247.436741:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177247.436743:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177247.436746:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000040:3.0:1786177247.436748:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177247.436753:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235593600, portal 25 00000100:00000001:2.0:1786177247.436755:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1786177247.436756:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.436757:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280810880 : -110371428740736 : ffff9b9e26325180) 00000100:00080000:3.0:1786177247.436759:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177247.436762:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:2.0:1786177247.436763:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.436770:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1786177247.436772:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:2.0:1786177247.436773:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177247.436774:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1786177247.436776:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000001:3.0:1786177247.436776:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177247.436779:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11921 00000100:00000200:2.0:1786177247.436780:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235593600, offset 0 00000400:00000200:3.0:1786177247.436783:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177247.436785:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:2.0:1786177247.436786:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177247.436787:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177247.436790:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436792:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177247.436794:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177247.436794:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436797:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177247.436799:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:2.0:1786177247.436800:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000040:3.0:1786177247.436800:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177247.436802:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177247.436804:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:2.0:1786177247.436805:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1786177247.436806:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177247.436808:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786177247.436810:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:3.0:1786177247.436812:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436813:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177247.436814:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1786177247.436816:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177247.436817:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436818:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177247.436820:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:2.0:1786177247.436821:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426704:426704:256:4294967295] 192.168.202.132@tcp LPNI seq info [426666:426666:8:4294967295] 00000100:00080000:3.0:1786177247.436823:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436825:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177247.436826:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:3.0:1786177247.436827:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177247.436829:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177247.436831:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177247.436833:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177247.436835:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1786177247.436836:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000008:00000001:3.0:1786177247.436838:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177247.436840:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177247.436843:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000010:3.0:1786177247.436843:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786177247.436845:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177247.436860:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177247.436861:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1786177247.436862:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000100:00000001:3.0:1786177247.436863:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177247.436865:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177247.436866:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177247.436869:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000010:3.0:1786177247.436870:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000800:00000200:2.0:1786177247.436873:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000100:00000001:3.0:1786177247.436873:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177247.436878:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1786177247.436878:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177247.436883:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000100:00000001:3.0:1786177247.436886:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000100:2.0:1786177247.436888:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436889:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1786177247.436891:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000100:00000040:3.0:1786177247.436893:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177247.436897:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1786177247.436900:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177247.436902:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1786177247.436903:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1786177247.436907:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177247.436910:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00080000:3.0:1786177247.436913:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177247.436914:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786177247.436914:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177247.436916:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1786177247.436918:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177247.436919:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177247.436922:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11921 00000400:00000200:3.0:1786177247.436926:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177247.436928:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1786177247.436929:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000040:2.0:1786177247.436932:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177247.436932:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177247.436935:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436937:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177247.436939:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177247.436941:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177247.436941:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177247.436943:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1786177247.436944:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177247.436945:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177247.436946:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1786177247.436948:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1786177247.436948:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177247.436950:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000200:2.0:1786177247.436952:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:3.0:1786177247.436953:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:2.0:1786177247.436955:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306598144 00000100:00000040:3.0:1786177247.436956:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436958:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:2.0:1786177247.436960:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c4e0. 00000100:00000040:3.0:1786177247.436960:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436962:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177247.436964:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:2.0:1786177247.436966:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1786177247.436966:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177247.436969:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177247.436970:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177247.436973:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306598144 02000000:00000001:3.0:1786177247.436973:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177247.436975:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177247.436976:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cae0. 00000100:00000001:3.0:1786177247.436977:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177247.436980:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000008:00100000:3.0:1786177247.436980:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1786177247.436983:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177247.436986:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1786177247.436988:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.436989:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1786177247.436990:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177247.436993:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177247/real 1786177247] req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1786177247.436994:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786177247.436996:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177247.436999:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177247.437000:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177247.437002:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177247.437003:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:2.0:1786177247.437004:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177247.437005:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1786177247.437005:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177247.437008:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 02000000:00000010:3.0:1786177247.437009:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000400:00000200:2.0:1786177247.437010:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000100:00000001:3.0:1786177247.437011:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177247.437012:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177247.437015:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1786177247.437016:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1786177247.437019:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177247.437024:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177247.437027:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177247.437029:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177247.437030:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437037:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177247.437039:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177247.437039:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.437042:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1786177247.437042:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786177247.437044:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177247.437045:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1786177247.437048:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177247.437053:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177247.437055:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177247.437056:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177247.437058:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:2.0:1786177247.437060:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177247.437061:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11921 00000400:00000200:3.0:1786177247.437065:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177247.437068:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177247.437069:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1786177247.437070:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177247.437072:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437073:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1786177247.437073:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177247.437075:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177247.437075:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177247.437077:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177247.437078:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1786177247.437079:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177247.437080:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1786177247.437081:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00080000:2.0:1786177247.437083:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:3.0:1786177247.437083:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177247.437085:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00080000:2.0:1786177247.437087:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000020:00000040:3.0:1786177247.437087:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177247.437089:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177247.437090:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:3.0:1786177247.437091:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177247.437094:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:2.0:1786177247.437095:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177247.437096:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177247.437098:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177247.437100:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177247.437103:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00100000:2.0:1786177247.437105:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235593600:192.168.202.132@tcp:250:kworker.0 00000100:00080000:3.0:1786177247.437105:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177247.437108:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177247.437109:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177247.437111:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1786177247.437112:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177247.437113:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1786177247.437114:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177247.437116:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325180 x1872930235593600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177247.437121:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177247.437123:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177247.437126:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177247.437128:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.437131:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000010:3.0:1786177247.437132:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786177247.437133:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177247.437134:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177247.437136:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:3.0:1786177247.437136:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177247.437137:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.437139:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1786177247.437139:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177247.437140:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1786177247.437141:0:2392:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:3.0:1786177247.437142:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177247.437143:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177247.437145:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:3.0:1786177247.437145:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 02000000:00000010:2.0:1786177247.437147:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786177247.437148:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.437152:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00100000:3.0:1786177247.437152:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1786177247.437153:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177247.437155:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325180. 00000100:00000001:2.0:1786177247.437159:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177247.437160:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437162:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177247.437166:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177247.437168:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.437170:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1786177247.437171:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437172:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.437174:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177247.437176:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177247.437177:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177247.437180:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177247.437181:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177247.437184:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11932) 00000100:00000001:2.0:1786177247.437188:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177247.437189:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.437191:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177247.437193:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177247.437195:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177247.437198:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593728:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177247.437201:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177247.437203:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177247.437205:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.437207:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177247.437209:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177247.437211:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177247.437215:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177247.437218:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177247.437222:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177247.437225:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235593728, portal 10 00000100:00000001:2.0:1786177247.437227:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177247.437228:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818048 : -110371428733568 : ffff9b9e26326d80) 00000100:00000040:2.0:1786177247.437235:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437239:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177247.437241:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177247.437242:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177247.437244:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235593728, offset 0 00000400:00000200:2.0:1786177247.437247:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177247.437250:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177247.437254:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177247.437257:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177247.437260:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177247.437262:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177247.437265:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177247.437268:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426705:426705:256:4294967295] 192.168.202.132@tcp LPNI seq info [426667:426667:8:4294967295] 00000400:00000200:2.0:1786177247.437283:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177247.437290:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177247.437295:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177247.437299:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177247.437303:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177247.437306:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786177247.437310:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177247.437312:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786177247.437316:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177247.437318:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177247.437321:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177247.437325:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177247.437328:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177247.437330:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177247.437333:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177247.437337:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437346:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177247.437350:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437392:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.437394:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177247.437397:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177247.437400:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177247.437403:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177247.437404:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306594308 00000100:00000001:2.0:1786177247.437406:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306598144 00000100:00000001:2.0:1786177247.437408:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306594308 00000100:00000001:2.0:1786177247.437409:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306598144 00000100:00000001:2.0:1786177247.437410:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177247.437412:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.437413:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177247.437416:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177247/real 1786177247] req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177247.437422:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177247.437423:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177247.437424:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177247.437426:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177247.437427:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177247.437428:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177247.437430:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177247.437436:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437445:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177247.437446:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437447:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177247.437448:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177247.437450:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437454:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177247.437457:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437461:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177247.437462:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437463:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177247.437464:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437465:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177247.437467:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177247.437469:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177247.437471:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177247.437473:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177247.437475:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177247.437480:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593728:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177247.437482:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177247.437484:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437488:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177247.437489:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.437491:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177247.437493:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177247.437494:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177247.437496:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593856:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177247.437498:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177247.437499:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177247.437500:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.437502:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177247.437504:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177247.437505:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177247.437508:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177247.437509:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177247.437511:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177247.437513:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235593856, portal 10 00000100:00000001:2.0:1786177247.437514:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177247.437515:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280820736 : -110371428730880 : ffff9b9e26327800) 00000100:00000040:2.0:1786177247.437519:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437523:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177247.437524:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177247.437526:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177247.437527:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235593856, offset 0 00000400:00000200:2.0:1786177247.437530:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177247.437532:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177247.437535:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177247.437537:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177247.437539:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177247.437541:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177247.437542:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177247.437544:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426706:426706:256:4294967295] 192.168.202.132@tcp LPNI seq info [426668:426668:8:4294967295] 00000400:00000200:2.0:1786177247.437547:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177247.437552:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177247.437555:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177247.437558:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177247.437560:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177247.437562:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786177247.437565:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177247.437567:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786177247.437570:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177247.437571:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177247.437574:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177247.437577:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177247.437581:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177247.437582:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177247.437585:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177247.437588:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437595:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177247.437597:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437604:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.437606:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177247.437609:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177247.437611:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177247.437613:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177247.437615:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306594308 00000100:00000001:2.0:1786177247.437617:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306598144 00000100:00000001:2.0:1786177247.437618:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306594308 00000100:00000001:2.0:1786177247.437620:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306598144 00000100:00000001:2.0:1786177247.437621:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177247.437622:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.437623:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177247.437626:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177247/real 1786177247] req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177247.437631:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177247.437632:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177247.437634:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177247.437635:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177247.437636:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177247.437637:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177247.437639:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177247.437645:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437649:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177247.437649:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437651:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177247.437651:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177247.437654:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437657:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177247.437660:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437664:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177247.437666:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437667:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177247.437668:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437669:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177247.437671:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177247.437673:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177247.437676:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177247.437678:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177247.437683:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177247.437691:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593856:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177247.437694:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177247.437696:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437700:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177247.437701:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.437703:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177247.437704:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177247.437705:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177247.437707:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593984:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177247.437709:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177247.437710:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177247.437711:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.437713:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177247.437715:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177247.437716:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177247.437718:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177247.437720:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177247.437722:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177247.437724:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235593984, portal 4 00000100:00000001:2.0:1786177247.437726:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177247.437727:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280814464 : -110371428737152 : ffff9b9e26325f80) 00000100:00000040:2.0:1786177247.437731:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437735:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177247.437736:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177247.437738:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177247.437739:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235593984, offset 0 00000400:00000200:2.0:1786177247.437742:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177247.437745:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177247.437747:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177247.437749:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177247.437751:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177247.437753:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177247.437755:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177247.437758:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426707:426707:256:4294967295] 192.168.202.132@tcp LPNI seq info [426669:426669:8:4294967295] 00000400:00000200:2.0:1786177247.437763:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177247.437769:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177247.437774:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177247.437777:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177247.437781:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177247.437784:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786177247.437788:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177247.437791:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786177247.437795:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177247.437797:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177247.437800:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177247.437804:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177247.437807:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177247.437809:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177247.437812:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177247.437816:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437824:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177247.437826:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437830:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.437831:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177247.437834:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177247.437836:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177247.437840:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177247.437842:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306594308 00000100:00000001:2.0:1786177247.437844:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306598144 00000100:00000001:2.0:1786177247.437859:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306594308 00000100:00000001:2.0:1786177247.437861:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306598144 00000100:00000001:2.0:1786177247.437863:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177247.437865:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.437867:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177247.437871:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177247/real 1786177247] req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177247.437878:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177247.437880:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177247.437881:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177247.437884:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177247.437885:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177247.437887:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177247.437890:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177247.437898:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437904:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177247.437906:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437907:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177247.437909:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177247.437912:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437918:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177247.437921:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437931:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177247.437934:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437935:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177247.437936:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437938:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177247.437939:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177247.437941:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177247.437944:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177247.437946:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177247.437950:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177247.437957:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235593984:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177247.437960:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437962:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177247.437965:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326d80 x1872930235593728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.437970:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177247.437972:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177247.437974:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177247.437977:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177247.437979:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177247.437980:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177247.437982:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177247.437984:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8400. 02000000:00000001:2.0:1786177247.437986:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177247.437987:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177247.437989:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326d80. 00000100:00000001:2.0:1786177247.437991:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177247.437993:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.437994:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177247.437997:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327800 x1872930235593856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.438009:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177247.438011:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177247.438012:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177247.438014:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177247.438015:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177247.438017:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177247.438019:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177247.438020:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8c00. 02000000:00000001:2.0:1786177247.438022:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177247.438023:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177247.438024:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327800. 00000100:00000001:2.0:1786177247.438027:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177247.438028:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.438029:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177247.438032:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325f80 x1872930235593984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.438038:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177247.438039:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177247.438040:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177247.438042:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177247.438044:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177247.438045:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177247.438047:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177247.438048:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab800. 02000000:00000001:2.0:1786177247.438050:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177247.438051:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177247.438053:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325f80. 00000100:00000001:2.0:1786177247.438055:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177247.438057:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.438059:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.438062:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177247.438064:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.438065:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177247.438067:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177247.438068:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177247.438072:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.438077:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177247.438078:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.438080:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177247.438081:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177247.438082:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177247.438084:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235594112:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177247.438086:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177247.438087:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177247.438089:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177247.438090:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177247.438092:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177247.438093:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177247.438096:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177247.438097:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177247.438100:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177247.438102:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235594112, portal 4 00000100:00000001:2.0:1786177247.438103:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177247.438104:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280819840 : -110371428731776 : ffff9b9e26327480) 00000100:00000040:2.0:1786177247.438108:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.438112:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177247.438114:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177247.438115:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177247.438117:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235594112, offset 0 00000400:00000200:2.0:1786177247.438119:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177247.438123:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177247.438126:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177247.438129:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177247.438132:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177247.438134:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177247.438136:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177247.438140:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426708:426708:256:4294967295] 192.168.202.132@tcp LPNI seq info [426670:426670:8:4294967295] 00000400:00000200:2.0:1786177247.438144:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177247.438151:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177247.438156:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177247.438160:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177247.438164:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177247.438167:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786177247.438172:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177247.438174:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786177247.438177:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177247.438178:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177247.438181:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177247.438183:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177247.438185:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177247.438187:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177247.438188:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177247.438191:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.438197:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177247.438198:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177272 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.438203:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.438204:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177247.438206:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177247.438208:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177247.438210:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177247.438212:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306594310 00000100:00000001:2.0:1786177247.438214:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306598144 00000100:00000001:2.0:1786177247.438215:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306594310 00000100:00000001:2.0:1786177247.438217:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306598144 00000100:00000001:2.0:1786177247.438218:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177247.438220:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.438221:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177247.438223:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177247/real 1786177247] req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177247.438228:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177247.438229:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177247.438230:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177247.438232:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177247.438233:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177247.438234:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177247.438236:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177247.438241:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.438245:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177247.438246:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.438247:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177247.438248:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177247.438250:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.438257:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177247.438261:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.438280:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177247.438284:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.438286:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177247.438287:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.438289:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177247.438291:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177247.438294:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177247.438298:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177247.438301:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177247.438305:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177247.438311:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235594112:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177247.438313:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.438315:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177247.438318:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327480 x1872930235594112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177272 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177247.438325:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177247.438326:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177247.438328:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177247.438331:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177247.438332:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177247.438333:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177247.438335:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177247.438337:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa400. 02000000:00000001:2.0:1786177247.438340:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177247.438341:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177247.438342:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327480. 00000100:00000001:2.0:1786177247.438345:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177247.438346:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.438348:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177247.438352:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177247.438353:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177247.438373:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177247.438375:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177247.948470:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177247.948479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177247.948483:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177247.948489:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177247.948494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177247.948498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177247.948500:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177247.948504:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177247.948507:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177248.332475:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177248.332481:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177248.332486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177248.332489:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177248.332491:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177248.332493:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177248.332498:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177248.332499:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177248.332503:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177248.332509:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177248.332514:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177248.332518:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177248.332527:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177248.332531:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177248.332536:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177248.332563:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177248.332567:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177248.332570:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177248.972438:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177248.972445:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177248.972448:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177248.972455:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177248.972459:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177248.972463:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177248.972466:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177248.972469:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177248.972473:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177249.356469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177249.356474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177249.356477:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177249.356480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177249.356480:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177249.356488:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177249.356490:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177249.356492:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177249.356494:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177249.356496:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177249.356500:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177249.356501:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177249.356503:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177249.356505:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177249.356509:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177249.356606:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177249.356611:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177249.356614:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177249.996479:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177249.996486:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177249.996488:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177249.996493:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177249.996496:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177249.996504:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177249.996507:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177249.996509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177249.996512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177250.380495:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177250.380502:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177250.380505:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177250.380510:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177250.380514:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177250.380517:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177250.380520:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177250.380522:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177250.380525:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177250.380646:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177250.380649:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177250.380653:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177250.380655:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177250.380658:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177250.380660:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177250.380742:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177250.380745:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177250.380748:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177251.020428:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177251.020436:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177251.020439:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177251.020445:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177251.020449:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177251.020453:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177251.020456:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177251.020460:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177251.020463:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177251.340490:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306598144, at 4306598208 00000100:00000040:1.0:1786177251.340490:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306598144, at 4306598208 00000100:00000010:0.0:1786177251.340507:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c4e0. 00000100:00000010:1.0:1786177251.340510:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cae0. 00000100:00000001:1.0:1786177251.404479:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177251.404485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177251.404486:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177251.404491:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177251.404492:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177251.404496:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177251.404499:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177251.404501:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177251.404504:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177251.404506:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177251.404509:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177251.404510:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177251.404515:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177251.404515:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177251.404517:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177251.404521:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177251.404525:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177251.404527:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177251.404530:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177251.404536:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177251.404538:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177251.404593:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177251.404597:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177251.404599:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177252.044469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177252.044476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177252.044479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177252.044485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177252.044490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177252.044494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177252.044496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177252.044499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177252.044502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177252.428463:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177252.428470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177252.428472:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177252.428472:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177252.428477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177252.428479:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177252.428484:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177252.428486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177252.428487:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177252.428489:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177252.428491:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177252.428495:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177252.428497:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177252.428498:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177252.428501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177252.428562:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177252.428565:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177252.428568:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177252.556508:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177252.556520:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177252.556523:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177252.556527:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177252.556532:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177252.556539:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11927 00000400:00000200:1.0:1786177252.556549:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177252.556555:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177252.556559:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177252.556562:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177252.556567:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177252.556569:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177252.556572:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177252.556576:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177252.556578:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177252.556582:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177252.556585:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177252.556588:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177252.556591:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177252.556594:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177252.556597:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177252.556599:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177252.556601:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177252.556605:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177252.556608:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177252.556611:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177252.556613:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177252.556617:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177252.556620:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177252.556623:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177252.556625:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177252.556635:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8700. 00000020:00000040:1.0:1786177252.556638:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177252.556643:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177252.556644:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177252.556648:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177252.556651:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177252.556655:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177252.556664:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad000. 00000100:00000001:1.0:1786177252.556670:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177252.556680:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177252.556692:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177252.556697:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177252.556702:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007a756761] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177252.556729:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177252.556733:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177252.556737:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177252.556738:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177252.556740:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177252.556744:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177252.556747:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11927 00000400:00000200:1.0:1786177252.556751:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177252.556755:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177252.556757:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177252.556761:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177252.556764:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177252.556767:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177252.556770:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177252.556773:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177252.556774:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1786177252.556775:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177252.556779:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1786177252.556779:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177252.556782:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1786177252.556784:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177252.556789:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177252.556790:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177252.556792:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177252.556811:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1786177252.556815:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177252.556815:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177252.556819:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1786177252.556820:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1786177252.556823:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1786177252.556824:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:2.0:1786177252.556826:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177252.556831:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00100000:2.0:1786177252.556833:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235594240:192.168.202.132@tcp:250:kworker.0 00000020:00000040:1.0:1786177252.556838:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:2.0:1786177252.556839:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177252.556843:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1786177252.556846:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:2.0:1786177252.556846:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177252.556851:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1786177252.556853:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177252.556857:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:2.0:1786177252.556857:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177252.556860:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177252.556863:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1786177252.556867:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177252.556869:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177252.556872:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000100:00000001:1.0:1786177252.556875:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:2.0:1786177252.556878:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000008:00100000:1.0:1786177252.556881:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000200:2.0:1786177252.556885:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235594240, portal 25 00000008:00000001:1.0:1786177252.556886:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.556888:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177252.556891:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.556891:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487983872 : -110371221567744 : ffff9b9e328b8700) 00000100:00000010:1.0:1786177252.556900:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8e00. 00000100:00000040:2.0:1786177252.556900:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177252.556907:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177252.556914:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177252.556918:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.556919:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1786177252.556924:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:2.0:1786177252.556924:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177252.556927:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 02000000:00000001:1.0:1786177252.556928:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1786177252.556932:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235594240, offset 0 02000000:00000001:1.0:1786177252.556933:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177252.556940:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 02000000:00000010:1.0:1786177252.556941:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261afc00. 00000100:00000001:1.0:1786177252.556948:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177252.556948:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177252.556957:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00100000:1.0:1786177252.556958:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786177252.556961:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1786177252.556971:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1786177252.556974:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:1.0:1786177252.556976:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177252.556980:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177252.556985:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000040:1.0:1786177252.556986:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ba93daf1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177252.556989:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426709:426709:256:4294967295] 192.168.202.132@tcp LPNI seq info [426671:426671:8:4294967295] 00000400:00000200:2.0:1786177252.556998:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177252.557008:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177252.557010:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00080000:1.0:1786177252.557015:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1786177252.557021:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00080000:1.0:1786177252.557022:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177252.557026:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177252.557028:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:2.0:1786177252.557030:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000100:00000001:1.0:1786177252.557032:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177252.557038:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11927 00000800:00000200:2.0:1786177252.557042:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:1.0:1786177252.557048:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:2.0:1786177252.557049:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000400:00000200:1.0:1786177252.557055:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000200:2.1:1786177252.557060:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:1.0:1786177252.557061:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:2.1:1786177252.557064:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000100:00000040:1.0:1786177252.557069:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000100:2.0:1786177252.557072:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177252.557076:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:2.0:1786177252.557077:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000100:00000040:1.0:1786177252.557083:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177252.557083:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1786177252.557088:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177252.557092:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000020:00000001:1.0:1786177252.557094:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:2.0:1786177252.557097:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000020:00000040:1.0:1786177252.557098:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000400:00000010:2.0:1786177252.557100:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000020:00000001:1.0:1786177252.557102:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1786177252.557104:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:1.0:1786177252.557106:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000100:00000200:2.0:1786177252.557110:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177252.557111:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177252.557118:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000001:2.0:1786177252.557125:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177252.557126:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:2.0:1786177252.557131:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177252.557132:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177252.557139:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177252.557145:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177252.557152:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177252.557154:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00000001:2.0:1786177252.557155:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1786177252.557160:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177252.557161:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177252.557166:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177252.557168:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177252.557171:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177252.557172:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306603264 02000000:00000001:1.0:1786177252.557177:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1786177252.557178:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c7e0. 02000000:00000001:1.0:1786177252.557183:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557186:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177252.557187:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177252.557191:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1786177252.557194:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306603264 00000008:00000001:1.0:1786177252.557195:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177252.557199:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c720. 00000100:00000001:1.0:1786177252.557200:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557203:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177252.557207:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1786177252.557208:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9c00. 00000100:00000001:2.0:1786177252.557209:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177252.557214:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000001:2.0:1786177252.557214:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177252.557220:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177252/real 1786177252] req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177252.557221:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177252.557225:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177252.557230:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177252.557233:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:2.0:1786177252.557233:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1786177252.557236:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1786177252.557236:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177252.557239:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177252.557242:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177252.557243:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 02000000:00000010:1.0:1786177252.557244:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af400. 00000100:00000001:2.0:1786177252.557246:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1786177252.557251:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177252.557253:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177252.557263:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1786177252.557270:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177252.557276:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177252.557281:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557286:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177252.557290:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177252.557291:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3ea07ed] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.557294:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177252.557298:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177252.557307:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177252.557309:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177252.557316:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1786177252.557318:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1786177252.557324:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000040:2.0:1786177252.557327:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177252.557328:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177252.557332:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177252.557338:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177252.557340:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786177252.557341:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.557344:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177252.557346:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1786177252.557347:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11927 00000100:00000001:2.0:1786177252.557348:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177252.557351:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:1.0:1786177252.557386:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:2.0:1786177252.557390:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:1.0:1786177252.557392:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177252.557396:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00080000:1.0:1786177252.557399:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177252.557399:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177252.557405:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177252.557408:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177252.557414:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177252.557421:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00100000:2.0:1786177252.557421:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235594240:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177252.557429:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177252.557430:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1786177252.557436:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177252.557436:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1786177252.557440:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000040:2.0:1786177252.557441:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8700 x1872930235594240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177252.557445:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177252.557451:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:2.0:1786177252.557451:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177252.557454:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:1.0:1786177252.557456:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 02000000:00000010:2.0:1786177252.557457:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 00000020:00000001:1.0:1786177252.557463:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 02000000:00000001:2.0:1786177252.557464:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177252.557466:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177252.557468:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000040:1.0:1786177252.557471:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000020:00000001:2.0:1786177252.557472:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177252.557475:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad000. 00000100:00000001:1.0:1786177252.557476:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1786177252.557479:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:1.0:1786177252.557482:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:2.0:1786177252.557483:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177252.557485:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8700. 00000100:00000001:1.0:1786177252.557488:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177252.557489:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177252.557492:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177252.557494:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:2.0:1786177252.557495:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1786177252.557501:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177252.557502:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177252.557505:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177252.557506:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557508:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177252.557511:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1786177252.557512:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177252.557513:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1786177252.557518:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1786177252.557518:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177252.557524:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557530:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177252.557531:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177252.557532:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177252.557536:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1786177252.557537:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 02000000:00000001:2.0:1786177252.557538:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177252.557542:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1786177252.557544:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177252.557546:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235594368:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177252.557549:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557550:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177252.557554:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177252.557555:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177252.557556:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9f80. 02000000:00000001:2.0:1786177252.557557:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177252.557561:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 00000020:00000040:1.0:1786177252.557562:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177252.557564:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177252.557569:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1786177252.557570:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177252.557573:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177252.557574:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000100:00000001:1.0:1786177252.557578:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1786177252.557579:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 02000000:00000001:1.0:1786177252.557582:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1786177252.557584:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235594368, portal 10 02000000:00000001:1.0:1786177252.557586:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557587:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177252.557589:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487985664 : -110371221565952 : ffff9b9e328b8e00) 02000000:00000010:1.0:1786177252.557595:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae000. 00000100:00000040:2.0:1786177252.557596:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177252.557602:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557604:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177252.557608:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177252.557612:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00100000:1.0:1786177252.557613:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1786177252.557615:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235594368, offset 0 00000400:00000200:2.0:1786177252.557623:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177252.557629:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177252.557634:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:1.0:1786177252.557636:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1786177252.557638:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1786177252.557642:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177252.557642:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177252.557647:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177252.557652:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000040:1.0:1786177252.557654:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4f3e5a2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177252.557655:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426710:426710:256:4294967295] 192.168.202.132@tcp LPNI seq info [426672:426672:8:4294967295] 00000400:00000200:2.0:1786177252.557664:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177252.557675:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:1.0:1786177252.557683:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177252.557683:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177252.557686:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000100:00080000:1.0:1786177252.557688:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1786177252.557691:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177252.557696:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000100:00080000:1.0:1786177252.557698:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000200:2.1:1786177252.557702:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177252.557703:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:2.1:1786177252.557705:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000100:00080000:1.0:1786177252.557707:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1786177252.557711:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177252.557715:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000100:00000001:1.0:1786177252.557716:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177252.557718:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:1.0:1786177252.557723:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11927 00000400:00000200:2.0:1786177252.557723:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177252.557728:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177252.557730:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000400:00000200:1.0:1786177252.557734:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177252.557735:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177252.557739:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177252.557744:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177252.557747:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1786177252.557750:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177252.557752:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177252.557758:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177252.557760:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557764:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177252.557765:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177252.557767:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177252.557771:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:2.0:1786177252.557774:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000100:00000001:1.0:1786177252.557777:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177252.557780:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:1.0:1786177252.557784:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177252.557785:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306599428 00000100:00000001:2.0:1786177252.557789:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306603264 00000020:00000040:1.0:1786177252.557790:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000001:2.0:1786177252.557791:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306599428 00000020:00000001:1.0:1786177252.557795:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1786177252.557795:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306603264 00000100:00000001:2.0:1786177252.557812:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177252.557815:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177252.557816:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:2.0:1786177252.557816:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000020:00000001:1.0:1786177252.557823:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00100000:2.0:1786177252.557823:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177252/real 1786177252] req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177252.557830:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000001:2.0:1786177252.557835:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177252.557837:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:1.0:1786177252.557840:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:2.0:1786177252.557840:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177252.557844:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000100:00000001:1.0:1786177252.557846:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1786177252.557847:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177252.557850:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1786177252.557854:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:2.0:1786177252.557855:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177252.557859:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177252.557866:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000200:2.0:1786177252.557867:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177252.557874:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177252.557875:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177252.557878:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177252.557880:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557881:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177252.557883:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177252.557886:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1786177252.557888:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177252.557891:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177252.557897:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557898:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177252.557901:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:2.0:1786177252.557903:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1786177252.557906:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177252.557912:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557913:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177252.557917:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557918:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.557921:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177252.557923:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1786177252.557925:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb480. 00000100:00000001:2.0:1786177252.557925:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177252.557927:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177252.557932:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000040:1.0:1786177252.557933:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00080000:2.0:1786177252.557935:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 02000000:00000001:1.0:1786177252.557940:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177252.557940:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177252.557944:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177252.557945:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177252.557951:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1786177252.557955:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235594368:192.168.202.132@tcp:38:kworker.0 02000000:00000001:1.0:1786177252.557956:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177252.557961:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.557962:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177252.557966:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177252.557970:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad400. 00000100:00000001:2.0:1786177252.557971:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177252.557974:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177252.557976:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1786177252.557977:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177252.557978:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177252.557984:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177252.557989:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235594496:192.168.202.132@tcp:38:kworker.0 00000100:00100000:1.0:1786177252.557990:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177252.557993:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177252.557996:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177252.557998:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177252.558001:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177252.558004:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177252.558008:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177252.558015:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177252.558020:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000100:00000001:1.0:1786177252.558022:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1786177252.558025:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000001:1.0:1786177252.558027:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177252.558031:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235594496, portal 10 00000100:00000040:1.0:1786177252.558036:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b192a357] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558036:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177252.558039:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487989248 : -110371221562368 : ffff9b9e328b9c00) 00000100:00000040:2.0:1786177252.558050:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177252.558052:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177252.558061:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11937) 00000100:00000001:2.0:1786177252.558061:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177252.558066:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177252.558068:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177252.558071:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235594496, offset 0 00000400:00000200:2.0:1786177252.558074:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177252.558078:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177252.558081:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177252.558083:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177252.558086:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177252.558088:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177252.558090:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177252.558092:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426711:426711:256:4294967295] 192.168.202.132@tcp LPNI seq info [426673:426673:8:4294967295] 00000400:00000200:2.0:1786177252.558095:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177252.558100:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177252.558103:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177252.558105:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177252.558108:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177252.558110:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177252.558113:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177252.558115:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177252.558117:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177252.558118:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177252.558120:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177252.558123:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177252.558125:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177252.558127:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177252.558129:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177252.558131:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558137:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177252.558139:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558143:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.558144:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177252.558146:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177252.558148:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177252.558150:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177252.558152:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306599428 00000100:00000001:2.0:1786177252.558153:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306603264 00000100:00000001:2.0:1786177252.558155:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306599428 00000100:00000001:2.0:1786177252.558156:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306603264 00000100:00000001:2.0:1786177252.558158:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177252.558159:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.558160:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177252.558162:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177252/real 1786177252] req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177252.558167:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177252.558168:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177252.558169:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177252.558170:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177252.558171:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177252.558173:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177252.558175:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177252.558180:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558184:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177252.558185:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558187:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177252.558188:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177252.558190:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558194:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177252.558196:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558200:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177252.558201:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558202:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177252.558203:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558204:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177252.558206:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177252.558207:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177252.558209:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177252.558210:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177252.558213:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177252.558217:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235594496:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177252.558219:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558221:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177252.558223:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8e00 x1872930235594368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558227:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177252.558229:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177252.558230:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177252.558232:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177252.558233:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177252.558234:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177252.558235:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177252.558236:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:2.0:1786177252.558238:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177252.558239:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177252.558240:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8e00. 00000100:00000001:2.0:1786177252.558242:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177252.558243:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558244:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177252.558247:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9c00 x1872930235594496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558254:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177252.558255:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177252.558257:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177252.558259:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177252.558260:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177252.558262:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177252.558264:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177252.558265:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af400. 02000000:00000001:2.0:1786177252.558267:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177252.558269:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177252.558270:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9c00. 00000100:00000001:2.0:1786177252.558273:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177252.558274:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558276:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558279:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177252.558281:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.558283:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177252.558285:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177252.558286:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177252.558291:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558298:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177252.558300:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177252.558302:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177252.558303:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177252.558304:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177252.558306:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235594624:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177252.558309:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177252.558310:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177252.558311:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177252.558312:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177252.558315:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177252.558316:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177252.558319:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177252.558320:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177252.558323:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177252.558325:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235594624, portal 4 00000100:00000001:2.0:1786177252.558326:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177252.558327:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487990144 : -110371221561472 : ffff9b9e328b9f80) 00000100:00000040:2.0:1786177252.558331:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558336:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177252.558338:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177252.558339:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177252.558341:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235594624, offset 0 00000400:00000200:2.0:1786177252.558343:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177252.558345:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177252.558348:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177252.558350:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177252.558352:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177252.558353:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177252.558355:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177252.558373:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426712:426712:256:4294967295] 192.168.202.132@tcp LPNI seq info [426674:426674:8:4294967295] 00000400:00000200:2.0:1786177252.558376:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177252.558381:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177252.558384:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177252.558386:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177252.558389:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177252.558391:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177252.558394:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177252.558396:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177252.558398:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177252.558400:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177252.558402:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177252.558404:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177252.558406:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177252.558407:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177252.558409:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177252.558412:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558417:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177252.558419:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558423:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.558424:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177252.558426:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177252.558429:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177252.558431:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177252.558432:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306599429 00000100:00000001:2.0:1786177252.558434:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306603264 00000100:00000001:2.0:1786177252.558435:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306599429 00000100:00000001:2.0:1786177252.558437:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306603264 00000100:00000001:2.0:1786177252.558438:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177252.558439:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.558440:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177252.558443:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177252/real 1786177252] req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177252.558449:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177252.558450:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177252.558451:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177252.558452:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177252.558453:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177252.558454:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177252.558456:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177252.558462:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558466:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177252.558467:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558468:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177252.558469:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177252.558471:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558475:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177252.558477:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558481:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177252.558482:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558483:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177252.558484:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558485:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177252.558486:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177252.558488:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177252.558490:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177252.558491:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177252.558494:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177252.558500:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235594624:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177252.558502:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177252.558504:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558508:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177252.558509:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177252.558510:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177252.558511:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177252.558512:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177252.558514:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235594752:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177252.558516:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177252.558517:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177252.558518:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177252.558519:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177252.558521:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177252.558522:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177252.558524:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177252.558525:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177252.558527:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177252.558529:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235594752, portal 4 00000100:00000001:2.0:1786177252.558530:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177252.558531:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487995520 : -110371221556096 : ffff9b9e328bb480) 00000100:00000040:2.0:1786177252.558535:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558539:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177252.558541:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177252.558542:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177252.558544:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235594752, offset 0 00000400:00000200:2.0:1786177252.558547:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177252.558549:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177252.558551:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177252.558553:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177252.558554:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177252.558556:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177252.558558:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177252.558560:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426713:426713:256:4294967295] 192.168.202.132@tcp LPNI seq info [426675:426675:8:4294967295] 00000400:00000200:2.0:1786177252.558562:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177252.558567:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177252.558570:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177252.558571:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177252.558574:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177252.558576:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177252.558578:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177252.558580:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177252.558582:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177252.558584:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177252.558586:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177252.558588:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177252.558590:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177252.558591:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177252.558592:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177252.558594:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558599:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177252.558601:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177277 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558604:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.558605:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177252.558607:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177252.558609:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177252.558610:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177252.558612:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306599431 00000100:00000001:2.0:1786177252.558614:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306603264 00000100:00000001:2.0:1786177252.558615:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306599431 00000100:00000001:2.0:1786177252.558616:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306603264 00000100:00000001:2.0:1786177252.558618:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177252.558619:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.558620:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177252.558622:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177252/real 1786177252] req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177252.558626:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177252.558627:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177252.558628:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177252.558629:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177252.558630:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177252.558632:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177252.558634:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177252.558639:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558643:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177252.558644:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558645:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177252.558645:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177252.558647:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558651:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177252.558653:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558657:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177252.558660:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558661:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177252.558661:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558662:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177252.558664:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177252.558665:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177252.558667:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177252.558668:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177252.558670:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177252.558675:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235594752:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177252.558677:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558678:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177252.558680:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9f80 x1872930235594624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558684:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177252.558685:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177252.558686:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177252.558687:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177252.558688:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177252.558689:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177252.558690:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177252.558691:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:2.0:1786177252.558692:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177252.558693:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177252.558694:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9f80. 00000100:00000001:2.0:1786177252.558697:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177252.558698:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558699:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177252.558701:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb480 x1872930235594752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177277 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177252.558709:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177252.558710:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177252.558711:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177252.558713:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177252.558713:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177252.558714:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177252.558715:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177252.558716:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:2.0:1786177252.558717:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177252.558717:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177252.558718:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb480. 00000100:00000001:2.0:1786177252.558720:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177252.558721:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558722:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177252.558724:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177252.558725:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177252.558726:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177252.558727:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177253.068474:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177253.068483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177253.068485:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177253.068491:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177253.068496:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177253.068499:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177253.068502:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177253.068505:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177253.068508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177253.452460:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177253.452468:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177253.452470:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177253.452474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177253.452476:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177253.452480:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177253.452481:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177253.452484:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177253.452486:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177253.452488:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177253.452490:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177253.452494:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1786177253.452494:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177253.452499:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177253.452505:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177253.452508:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177253.452511:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177253.452516:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177253.452518:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177253.452522:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177253.452524:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177253.452525:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177253.452527:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177253.452529:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177253.452530:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177253.452531:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177253.452533:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177253.452582:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177253.452584:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177253.452587:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177254.092445:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177254.092452:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177254.092455:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177254.092460:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177254.092465:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177254.092469:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177254.092472:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177254.092475:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177254.092478:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177254.476427:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177254.476431:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177254.476433:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177254.476435:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177254.476436:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177254.476442:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177254.476443:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177254.476445:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177254.476447:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177254.476449:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177254.476451:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177254.476451:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177254.476454:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177254.476457:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177254.476459:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177254.476507:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177254.476510:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177254.476512:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177255.116461:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177255.116470:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177255.116474:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177255.116480:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177255.116484:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177255.116488:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177255.116490:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177255.116493:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177255.116497:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177255.500452:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177255.500455:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177255.500458:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177255.500458:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177255.500460:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177255.500464:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177255.500464:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177255.500465:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177255.500467:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177255.500468:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177255.500469:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177255.500470:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177255.500473:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177255.500476:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177255.500479:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177255.500532:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177255.500535:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177255.500538:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177256.140467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177256.140476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177256.140479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177256.140485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177256.140489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177256.140493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177256.140497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177256.140499:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177256.140502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177256.460461:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306603264, at 4306603328 00000100:00000040:1.0:1786177256.460461:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306603264, at 4306603328 00000100:00000010:0.0:1786177256.460476:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c7e0. 00000100:00000010:1.0:1786177256.460477:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c720. 00000100:00000001:1.0:1786177256.524425:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177256.524436:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177256.524439:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177256.524446:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177256.524452:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177256.524456:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177256.524460:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177256.524463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177256.524466:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177256.524538:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177256.524545:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177256.524550:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177256.524552:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177256.524555:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177256.524556:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177256.524622:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177256.524626:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177256.524629:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177257.164514:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177257.164528:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177257.164531:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177257.164541:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177257.164548:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177257.164555:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177257.164559:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177257.164564:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177257.164568:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.548493:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177257.548500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177257.548502:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177257.548508:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.548509:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177257.548518:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177257.548519:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177257.548522:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.548528:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177257.548530:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177257.548533:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.548541:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177257.548545:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177257.548548:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177257.548551:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177257.548599:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177257.548602:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177257.548605:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177257.676442:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177257.676453:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177257.676456:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177257.676460:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177257.676464:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177257.676471:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11932 00000400:00000200:1.0:1786177257.676478:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177257.676483:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177257.676485:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177257.676487:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676490:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177257.676492:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676494:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177257.676496:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177257.676499:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177257.676501:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177257.676504:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177257.676505:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177257.676507:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177257.676509:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676510:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177257.676511:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676512:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177257.676514:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177257.676516:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676518:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177257.676520:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177257.676522:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177257.676524:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.676526:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177257.676527:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177257.676534:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328ba680. 00000020:00000040:1.0:1786177257.676537:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177257.676539:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177257.676541:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.676543:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177257.676545:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177257.676548:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177257.676554:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af000. 00000100:00000001:1.0:1786177257.676558:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177257.676565:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177257.676573:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177257.676576:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177257.676579:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000497a117c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177257.676597:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177257.676599:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177257.676601:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177257.676602:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177257.676603:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177257.676605:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177257.676607:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11932 00000400:00000200:1.0:1786177257.676609:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177257.676611:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177257.676613:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177257.676615:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676617:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177257.676618:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676619:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177257.676620:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1786177257.676621:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177257.676622:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177257.676623:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177257.676624:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177257.676625:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177257.676626:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177257.676628:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676631:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177257.676633:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676634:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177257.676635:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:2.0:1786177257.676636:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1786177257.676637:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676640:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.676641:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1786177257.676642:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177257.676647:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177257.676650:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.676651:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:2.0:1786177257.676652:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1786177257.676655:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177257.676659:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.676662:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177257.676666:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb100. 00000020:00000040:1.0:1786177257.676670:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000100:00000001:2.0:1786177257.676670:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1786177257.676674:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1786177257.676674:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177257.676675:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.676679:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177257.676680:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177257.676681:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177257.676682:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:1.0:1786177257.676684:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261aec00. 00000100:00000001:1.0:1786177257.676687:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177257.676687:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177257.676693:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177257.676694:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177257.676702:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1786177257.676702:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235594880:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177257.676705:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177257.676710:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f57195c8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.676711:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177257.676716:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1786177257.676717:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177257.676720:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177257.676721:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177257.676726:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177257.676727:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1786177257.676727:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1786177257.676730:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177257.676732:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177257.676733:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11932 02000000:00000010:2.0:1786177257.676735:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 00000400:00000200:1.0:1786177257.676736:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177257.676740:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 02000000:00000001:2.0:1786177257.676740:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177257.676743:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177257.676746:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676759:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177257.676761:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177257.676762:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676765:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177257.676767:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:2.0:1786177257.676767:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000020:00000040:1.0:1786177257.676769:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177257.676771:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177257.676772:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177257.676774:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177257.676775:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000010:2.0:1786177257.676776:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000040:1.0:1786177257.676778:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676781:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177257.676783:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676784:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177257.676785:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00000200:2.0:1786177257.676786:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235594880, portal 25 00000100:00080000:1.0:1786177257.676787:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676790:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177257.676792:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177257.676792:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1786177257.676795:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177257.676796:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487991936 : -110371221559680 : ffff9b9e328ba680) 02000000:00000001:1.0:1786177257.676797:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.676800:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177257.676802:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177257.676804:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.676806:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177257.676808:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1786177257.676809:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb800. 00000020:00000040:1.0:1786177257.676813:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177257.676816:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177257.676818:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.676821:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177257.676822:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177257.676824:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.676827:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:1.0:1786177257.676829:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad800. 00000100:00000040:2.0:1786177257.676832:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177257.676836:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000001:1.0:1786177257.676837:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177257.676843:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1786177257.676843:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235594880, offset 0 00000100:00000001:1.0:1786177257.676852:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177257.676855:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177257.676857:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000040:1.0:1786177257.676859:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008148c2a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177257.676865:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177257.676869:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1786177257.676869:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177257.676874:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177257.676875:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177257.676876:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177257.676879:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177257.676880:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00080000:1.0:1786177257.676883:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11932 00000400:00000200:2.0:1786177257.676888:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177257.676889:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177257.676892:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177257.676894:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177257.676896:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:1.0:1786177257.676898:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676900:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177257.676903:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1786177257.676904:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676908:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177257.676911:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:2.0:1786177257.676911:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000040:1.0:1786177257.676912:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177257.676915:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:2.0:1786177257.676919:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426714:426714:256:4294967295] 192.168.202.132@tcp LPNI seq info [426676:426676:8:4294967295] 00000020:00000040:1.0:1786177257.676921:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177257.676925:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177257.676927:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000200:2.0:1786177257.676928:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177257.676931:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676933:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177257.676935:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676936:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177257.676939:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:1.0:1786177257.676943:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676946:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177257.676948:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1786177257.676948:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 02000000:00000001:1.0:1786177257.676950:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177257.676954:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.676956:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177257.676957:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177257.676960:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177257.676960:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177257.676963:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177257.676967:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9180. 00000020:00000040:1.0:1786177257.676968:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000800:00000010:2.0:1786177257.676969:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 02000000:00000001:1.0:1786177257.676970:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177257.676973:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.676974:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177257.676975:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177257.676976:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177257.676980:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000010:1.0:1786177257.676982:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af800. 00000100:00000001:1.0:1786177257.676984:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177257.676985:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00100000:1.0:1786177257.676988:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.1:1786177257.676996:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177257.676997:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177257.677000:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786177257.677003:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000100:00000040:1.0:1786177257.677004:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d01dc3b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177257.677014:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1786177257.677014:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00080000:1.0:1786177257.677016:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:2.0:1786177257.677018:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000100:00080000:1.0:1786177257.677019:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177257.677021:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177257.677024:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177257.677025:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177257.677027:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11932 00000400:00000200:2.0:1786177257.677028:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177257.677030:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177257.677034:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177257.677036:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177257.677039:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:1.0:1786177257.677041:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.677044:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177257.677045:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.677047:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177257.677047:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000020:00000001:1.0:1786177257.677051:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000010:2.0:1786177257.677052:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000020:00000040:1.0:1786177257.677053:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177257.677055:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177257.677058:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:1.0:1786177257.677059:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:1.0:1786177257.677060:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:2.0:1786177257.677060:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1786177257.677064:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177257.677065:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177257.677069:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:2.0:1786177257.677069:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177257.677072:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177257.677074:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:1.0:1786177257.677077:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177257.677080:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177257.677082:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177257.677083:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177257.677084:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.677085:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177257.677086:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177257.677088:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.677091:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177257.677095:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8a80. 00000020:00000040:1.0:1786177257.677097:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000001:2.0:1786177257.677098:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1786177257.677099:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177257.677101:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.677102:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177257.677104:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177257.677105:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177257.677107:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177257.677108:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad400. 00000100:00000001:1.0:1786177257.677111:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177257.677117:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177257.677125:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177257.677126:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177257.677128:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.677130:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1786177257.677132:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002bb61190] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177257.677138:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.677138:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177257.677144:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11942) 00000400:00000200:2.0:1786177257.677148:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177257.677151:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306608384 00000100:00000010:2.0:1786177257.677157:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cd80. 00000100:00000001:2.0:1786177257.677163:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177257.677168:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306608384 00000100:00000010:2.0:1786177257.677170:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c3c0. 00000100:00000001:2.0:1786177257.677173:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177257.677176:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177257.677179:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.677181:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177257.677186:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177257/real 1786177257] req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177257.677194:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177257.677196:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177257.677198:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177257.677200:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177257.677201:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177257.677204:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177257.677207:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177257.677215:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677221:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177257.677222:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677224:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177257.677226:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177257.677229:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677235:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177257.677239:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677246:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177257.677248:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677249:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177257.677250:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677252:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177257.677254:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177257.677257:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177257.677260:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177257.677263:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177257.677266:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177257.677273:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235594880:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177257.677278:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677280:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177257.677282:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328ba680 x1872930235594880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677288:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177257.677290:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177257.677292:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177257.677294:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177257.677296:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177257.677297:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177257.677299:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177257.677301:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af000. 02000000:00000001:2.0:1786177257.677305:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177257.677306:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177257.677308:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328ba680. 00000100:00000001:2.0:1786177257.677311:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177257.677312:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677315:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677321:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177257.677323:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.677325:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177257.677326:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177257.677327:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177257.677331:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677336:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177257.677338:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177257.677340:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177257.677341:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177257.677344:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177257.677347:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595008:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177257.677350:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177257.677352:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177257.677353:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177257.677355:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177257.677383:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177257.677385:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177257.677388:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177257.677391:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177257.677394:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177257.677397:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235595008, portal 10 00000100:00000001:2.0:1786177257.677399:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177257.677400:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487994624 : -110371221556992 : ffff9b9e328bb100) 00000100:00000040:2.0:1786177257.677407:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677412:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177257.677415:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177257.677417:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177257.677420:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235595008, offset 0 00000400:00000200:2.0:1786177257.677424:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177257.677428:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177257.677431:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177257.677442:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177257.677444:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177257.677446:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177257.677449:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177257.677452:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426715:426715:256:4294967295] 192.168.202.132@tcp LPNI seq info [426677:426677:8:4294967295] 00000400:00000200:2.0:1786177257.677456:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177257.677462:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177257.677466:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177257.677469:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177257.677473:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177257.677476:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786177257.677480:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177257.677484:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1786177257.677487:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177257.677489:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177257.677492:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177257.677495:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177257.677498:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177257.677499:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177257.677501:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177257.677505:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677512:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177257.677514:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677520:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.677522:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177257.677524:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177257.677528:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177257.677531:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177257.677532:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306604548 00000100:00000001:2.0:1786177257.677535:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306608384 00000100:00000001:2.0:1786177257.677537:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306604548 00000100:00000001:2.0:1786177257.677539:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306608384 00000100:00000001:2.0:1786177257.677541:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177257.677543:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.677545:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177257.677548:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177257/real 1786177257] req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177257.677561:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177257.677562:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177257.677563:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177257.677565:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177257.677567:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177257.677568:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177257.677571:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177257.677579:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677585:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177257.677586:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677587:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177257.677589:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177257.677592:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677598:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177257.677601:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677607:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177257.677609:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677611:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177257.677612:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677613:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177257.677615:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177257.677618:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177257.677620:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177257.677622:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177257.677628:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177257.677636:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595008:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177257.677639:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177257.677643:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677649:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177257.677650:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177257.677652:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177257.677654:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177257.677656:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177257.677658:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595136:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177257.677661:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177257.677663:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177257.677664:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177257.677666:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177257.677668:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177257.677671:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177257.677673:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177257.677676:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177257.677679:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177257.677682:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235595136, portal 10 00000100:00000001:2.0:1786177257.677684:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177257.677685:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487996416 : -110371221555200 : ffff9b9e328bb800) 00000100:00000040:2.0:1786177257.677691:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677698:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177257.677700:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177257.677702:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177257.677704:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235595136, offset 0 00000400:00000200:2.0:1786177257.677708:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177257.677712:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177257.677714:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177257.677717:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177257.677719:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177257.677721:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177257.677723:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177257.677727:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426716:426716:256:4294967295] 192.168.202.132@tcp LPNI seq info [426678:426678:8:4294967295] 00000400:00000200:2.0:1786177257.677731:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177257.677736:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177257.677739:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177257.677742:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177257.677745:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177257.677760:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786177257.677765:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177257.677767:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1786177257.677771:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177257.677773:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177257.677777:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177257.677780:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177257.677782:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177257.677784:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177257.677786:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177257.677789:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677796:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177257.677800:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677805:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.677806:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177257.677809:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177257.677811:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177257.677814:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177257.677817:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306604548 00000100:00000001:2.0:1786177257.677819:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306608384 00000100:00000001:2.0:1786177257.677821:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306604548 00000100:00000001:2.0:1786177257.677823:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306608384 00000100:00000001:2.0:1786177257.677825:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177257.677826:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.677828:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177257.677832:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177257/real 1786177257] req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177257.677839:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177257.677840:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177257.677842:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177257.677844:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177257.677846:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177257.677848:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177257.677851:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177257.677859:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677864:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177257.677866:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677868:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177257.677869:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177257.677872:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677878:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177257.677882:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677888:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177257.677891:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677892:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177257.677893:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.677895:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177257.677896:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177257.677899:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177257.677902:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177257.677904:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177257.677907:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177257.677915:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595136:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177257.677918:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177257.677921:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677927:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177257.677929:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177257.677931:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177257.677933:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177257.677935:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177257.677938:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595264:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177257.677941:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177257.677943:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177257.677945:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177257.677946:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177257.677949:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177257.677951:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177257.677954:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177257.677956:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177257.677959:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177257.677962:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235595264, portal 4 00000100:00000001:2.0:1786177257.677964:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177257.677965:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487986560 : -110371221565056 : ffff9b9e328b9180) 00000100:00000040:2.0:1786177257.677970:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.677976:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177257.677978:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177257.677980:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177257.677983:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235595264, offset 0 00000400:00000200:2.0:1786177257.677987:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177257.677990:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177257.677994:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177257.677997:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177257.678000:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177257.678002:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177257.678004:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177257.678007:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426717:426717:256:4294967295] 192.168.202.132@tcp LPNI seq info [426679:426679:8:4294967295] 00000400:00000200:2.0:1786177257.678010:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177257.678016:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177257.678020:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177257.678024:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177257.678027:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177257.678030:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786177257.678034:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177257.678036:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1786177257.678039:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177257.678041:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177257.678044:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177257.678046:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177257.678049:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177257.678050:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177257.678052:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177257.678056:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678068:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177257.678070:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678076:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.678077:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177257.678080:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177257.678082:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177257.678085:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177257.678087:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306604548 00000100:00000001:2.0:1786177257.678089:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306608384 00000100:00000001:2.0:1786177257.678091:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306604548 00000100:00000001:2.0:1786177257.678093:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306608384 00000100:00000001:2.0:1786177257.678095:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177257.678097:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.678098:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177257.678102:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177257/real 1786177257] req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177257.678108:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177257.678109:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177257.678111:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177257.678114:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177257.678115:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177257.678118:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177257.678121:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177257.678129:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678134:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177257.678136:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678137:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177257.678138:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177257.678142:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678148:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177257.678151:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678158:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177257.678160:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678161:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177257.678163:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678164:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177257.678166:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177257.678168:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177257.678171:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177257.678173:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177257.678177:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177257.678183:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595264:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177257.678187:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177257.678189:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678195:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177257.678196:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177257.678198:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177257.678200:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177257.678202:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177257.678205:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595392:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177257.678207:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177257.678209:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177257.678211:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177257.678212:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177257.678215:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177257.678217:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177257.678220:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177257.678223:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177257.678226:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177257.678228:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235595392, portal 4 00000100:00000001:2.0:1786177257.678229:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177257.678231:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487984768 : -110371221566848 : ffff9b9e328b8a80) 00000100:00000040:2.0:1786177257.678236:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678241:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177257.678243:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177257.678245:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177257.678248:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235595392, offset 0 00000400:00000200:2.0:1786177257.678251:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177257.678254:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177257.678257:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177257.678260:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177257.678262:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177257.678264:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177257.678266:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177257.678268:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426718:426718:256:4294967295] 192.168.202.132@tcp LPNI seq info [426680:426680:8:4294967295] 00000400:00000200:2.0:1786177257.678273:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177257.678278:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177257.678282:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177257.678285:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177257.678289:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177257.678291:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1786177257.678295:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177257.678298:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1786177257.678301:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177257.678303:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177257.678305:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177257.678308:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177257.678311:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177257.678312:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177257.678314:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177257.678317:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678323:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177257.678326:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177282 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678332:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.678334:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177257.678336:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177257.678339:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177257.678342:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177257.678344:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306604550 00000100:00000001:2.0:1786177257.678346:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306608384 00000100:00000001:2.0:1786177257.678348:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306604550 00000100:00000001:2.0:1786177257.678350:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306608384 00000100:00000001:2.0:1786177257.678352:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177257.678354:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.678372:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177257.678376:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177257/real 1786177257] req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177257.678383:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177257.678384:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177257.678386:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177257.678388:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177257.678390:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177257.678392:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177257.678395:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177257.678402:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678408:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177257.678409:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678411:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177257.678412:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177257.678416:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678421:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177257.678425:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678430:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177257.678432:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678434:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177257.678435:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678436:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177257.678438:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177257.678440:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177257.678443:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177257.678445:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177257.678448:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177257.678460:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595392:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177257.678465:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678466:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177257.678469:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb100 x1872930235595008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678476:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177257.678477:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177257.678479:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177257.678482:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177257.678483:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177257.678484:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177257.678487:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177257.678488:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:2.0:1786177257.678491:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177257.678492:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177257.678493:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb100. 00000100:00000001:2.0:1786177257.678496:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177257.678497:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678499:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177257.678503:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb800 x1872930235595136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678509:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177257.678510:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177257.678511:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177257.678513:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177257.678515:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177257.678516:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177257.678518:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177257.678519:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:2.0:1786177257.678522:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177257.678523:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177257.678524:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb800. 00000100:00000001:2.0:1786177257.678527:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177257.678528:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678529:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177257.678532:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9180 x1872930235595264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678537:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177257.678538:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177257.678539:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177257.678541:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177257.678542:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177257.678543:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177257.678544:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177257.678546:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af800. 02000000:00000001:2.0:1786177257.678548:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177257.678549:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177257.678550:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9180. 00000100:00000001:2.0:1786177257.678552:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177257.678553:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678554:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177257.678558:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8a80 x1872930235595392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177282 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177257.678563:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177257.678564:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177257.678565:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177257.678567:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177257.678568:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177257.678569:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177257.678570:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177257.678572:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:2.0:1786177257.678573:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177257.678574:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177257.678575:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8a80. 00000100:00000001:2.0:1786177257.678577:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177257.678578:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678580:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177257.678584:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177257.678585:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177257.678588:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177257.678589:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177258.188489:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177258.188497:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177258.188500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177258.188506:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177258.188511:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177258.188515:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177258.188518:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177258.188521:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177258.188524:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177258.572457:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177258.572459:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177258.572464:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177258.572465:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177258.572468:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177258.572469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177258.572472:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177258.572472:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177258.572478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177258.572478:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177258.572481:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177258.572481:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177258.572485:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177258.572486:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177258.572488:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177258.572541:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177258.572546:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177258.572550:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177259.212488:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177259.212499:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177259.212502:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177259.212515:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177259.212520:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177259.212524:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177259.212526:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177259.212530:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177259.212533:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177259.596468:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177259.596474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177259.596476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177259.596479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177259.596480:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177259.596487:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177259.596487:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177259.596490:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177259.596495:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177259.596496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177259.596499:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177259.596500:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177259.596504:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177259.596507:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177259.596510:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177259.596600:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177259.596603:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177259.596607:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177260.236468:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177260.236477:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177260.236479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177260.236485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177260.236490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177260.236494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177260.236497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177260.236500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177260.236503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177260.620466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177260.620469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177260.620474:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177260.620475:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177260.620477:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177260.620481:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177260.620482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177260.620486:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177260.620487:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177260.620490:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177260.620491:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177260.620493:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177260.620496:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177260.620496:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177260.620498:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177260.620502:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177260.620505:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177260.620507:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177260.620509:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177260.620513:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177260.620515:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177260.620568:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177260.620571:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177260.620573:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177261.260454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177261.260462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177261.260464:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177261.260469:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177261.260473:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177261.260476:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177261.260479:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177261.260482:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177261.260485:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177261.580465:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306608384, at 4306608448 00000100:00000010:0.0:1786177261.580479:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cd80. 00000100:00000040:1.0:1786177261.581273:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306608384, at 4306608448 00000100:00000010:1.0:1786177261.581305:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c3c0. 00000100:00000001:1.0:1786177261.644483:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177261.644491:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177261.644494:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177261.644496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177261.644501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177261.644506:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177261.644507:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177261.644510:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177261.644512:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177261.644518:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177261.644523:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177261.644527:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177261.644530:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177261.644536:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177261.644539:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177261.644691:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177261.644695:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177261.644699:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177262.284458:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177262.284466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177262.284468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177262.284473:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177262.284477:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177262.284481:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177262.284483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177262.284486:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177262.284489:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177262.668463:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177262.668466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177262.668470:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177262.668473:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177262.668474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177262.668476:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177262.668477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177262.668483:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177262.668484:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177262.668485:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177262.668491:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177262.668495:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177262.668498:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177262.668501:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177262.668505:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177262.668541:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177262.668544:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177262.668546:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177262.796496:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177262.796507:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177262.796509:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177262.796513:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177262.796517:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177262.796532:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11937 00000400:00000200:3.0:1786177262.796540:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177262.796546:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177262.796548:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177262.796551:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796553:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177262.796555:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796556:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177262.796559:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177262.796560:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177262.796563:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177262.796565:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177262.796567:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177262.796568:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177262.796570:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796572:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177262.796573:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796574:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177262.796577:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177262.796581:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796583:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177262.796586:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177262.796589:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177262.796592:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177262.796594:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177262.796596:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177262.796606:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000020:00000040:3.0:1786177262.796610:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177262.796614:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177262.796616:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177262.796619:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177262.796621:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177262.796625:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177262.796631:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786177262.796635:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177262.796642:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177262.796649:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177262.796653:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177262.796656:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177262.796670:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177262.796672:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177262.796674:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177262.796675:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177262.796677:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177262.796678:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177262.796680:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11937 00000400:00000200:3.0:1786177262.796682:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177262.796685:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177262.796686:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177262.796688:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796689:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177262.796691:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796692:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177262.796693:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177262.796694:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177262.796695:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177262.796697:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177262.796724:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177262.796725:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1786177262.796726:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:2.0:1786177262.796731:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1786177262.796732:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177262.796736:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177262.796736:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177262.796741:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796745:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:2.0:1786177262.796747:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177262.796749:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177262.796756:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796760:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.796762:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1786177262.796764:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177262.796766:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177262.796770:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177262.796771:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177262.796774:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177262.796778:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177262.796782:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177262.796783:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235595520:192.168.202.132@tcp:250:kworker.0 00000100:00000001:3.0:1786177262.796786:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177262.796790:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1786177262.796792:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1786177262.796794:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1786177262.796795:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177262.796798:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177262.796799:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177262.796801:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 02000000:00000001:2.0:1786177262.796803:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1786177262.796805:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177262.796808:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177262.796809:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177262.796810:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:3.0:1786177262.796811:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177262.796814:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177262.796815:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177262.796816:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177262.796822:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000400:00000200:2.0:1786177262.796826:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177262.796827:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177262.796834:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000100:00100000:3.0:1786177262.796834:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1786177262.796844:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000001:3.0:1786177262.796845:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177262.796850:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177262.796853:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235595520, portal 25 00000100:00000040:3.0:1786177262.796854:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.796858:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177262.796860:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280807296 : -110371428744320 : ffff9b9e26324380) 00000100:00000040:2.0:1786177262.796868:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177262.796879:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.796884:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1786177262.796885:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177262.796889:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:2.0:1786177262.796891:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796891:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177262.796894:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177262.796896:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:2.0:1786177262.796897:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00080000:3.0:1786177262.796899:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11937 00000100:00000200:2.0:1786177262.796903:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235595520, offset 0 00000400:00000200:3.0:1786177262.796905:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177262.796911:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:2.0:1786177262.796915:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177262.796915:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177262.796921:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796924:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177262.796927:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177262.796928:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796933:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177262.796935:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:2.0:1786177262.796938:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000040:3.0:1786177262.796938:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177262.796941:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177262.796945:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000400:00000200:2.0:1786177262.796946:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1786177262.796948:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177262.796952:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:2.0:1786177262.796953:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:3.0:1786177262.796958:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796960:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1786177262.796961:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1786177262.796963:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796965:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177262.796966:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177262.796968:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177262.796969:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177262.796970:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177262.796972:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177262.796975:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177262.796976:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177262.796978:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1786177262.796980:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000008:00000001:3.0:1786177262.796981:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177262.796984:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177262.796987:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000400:00000200:2.0:1786177262.796988:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426719:426719:256:4294967295] 192.168.202.132@tcp LPNI seq info [426681:426681:8:4294967295] 00000020:00000040:3.0:1786177262.796990:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177262.796994:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177262.796997:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177262.797001:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177262.797001:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177262.797004:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177262.797007:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177262.797011:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000400:00000200:2.0:1786177262.797016:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:3.0:1786177262.797017:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177262.797026:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.0:1786177262.797027:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177262.797036:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000100:00000001:3.0:1786177262.797038:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177262.797041:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177262.797045:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:3.0:1786177262.797047:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1786177262.797052:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000001:3.0:1786177262.797058:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177262.797063:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.1:1786177262.797063:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177262.797070:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00080000:3.0:1786177262.797071:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177262.797074:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177262.797077:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1786177262.797078:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177262.797081:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177262.797087:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1786177262.797090:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177262.797094:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1786177262.797096:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11937 00000400:00000200:2.0:1786177262.797102:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000200:3.0:1786177262.797103:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177262.797107:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000400:00000200:3.0:1786177262.797108:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177262.797112:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177262.797114:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1786177262.797117:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000200:2.0:1786177262.797122:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177262.797122:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177262.797126:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.797131:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177262.797136:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177262.797139:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:2.0:1786177262.797140:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1786177262.797141:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:2.0:1786177262.797147:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177262.797151:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177262.797156:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177262.797162:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:2.0:1786177262.797163:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.797167:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1786177262.797167:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.797171:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177262.797174:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177262.797175:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177262.797181:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177262.797184:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306613504 00000100:00000001:3.0:1786177262.797185:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177262.797189:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000010:2.0:1786177262.797192:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290ccc0. 00000100:00080000:3.0:1786177262.797193:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177262.797197:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177262.797199:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177262.797201:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177262.797203:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1786177262.797208:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177262.797209:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177262.797212:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306613504 00000008:00100000:3.0:1786177262.797212:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177262.797217:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177262.797219:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c060. 00000100:00000001:3.0:1786177262.797219:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177262.797225:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000100:00000001:2.0:1786177262.797226:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177262.797231:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1786177262.797233:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:2.0:1786177262.797235:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177262.797236:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177262.797238:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.797241:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1786177262.797241:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177262.797242:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177262.797244:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177262.797248:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00100000:2.0:1786177262.797250:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177262/real 1786177262] req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177262.797251:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177262.797262:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1786177262.797264:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177262.797268:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1786177262.797271:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:2.0:1786177262.797272:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000100:00000001:3.0:1786177262.797274:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177262.797277:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000100:00000040:3.0:1786177262.797278:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1786177262.797282:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177262.797285:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1786177262.797290:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177262.797292:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177262.797293:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177262.797299:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177262.797301:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177262.797304:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1786177262.797311:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177262.797312:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177262.797317:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11937 00000400:00000200:3.0:1786177262.797326:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177262.797327:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177262.797330:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.797333:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177262.797335:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177262.797341:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.797348:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1786177262.797352:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000040:2.0:1786177262.797353:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177262.797388:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177262.797399:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177262.797402:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.797406:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177262.797410:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177262.797410:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177262.797415:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177262.797418:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177262.797420:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:2.0:1786177262.797422:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:3.0:1786177262.797425:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.797430:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00080000:2.0:1786177262.797431:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000020:00000001:3.0:1786177262.797434:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177262.797436:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:3.0:1786177262.797437:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000040:2.0:1786177262.797444:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177262.797453:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235595520:192.168.202.132@tcp:250:kworker.0 00000020:00000001:3.0:1786177262.797457:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:2.0:1786177262.797459:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1786177262.797461:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:2.0:1786177262.797463:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1786177262.797464:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177262.797468:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324380 x1872930235595520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177262.797469:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177262.797476:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.797480:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177262.797485:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:3.0:1786177262.797485:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177262.797488:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:2.0:1786177262.797490:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:3.0:1786177262.797493:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 02000000:00000010:2.0:1786177262.797495:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 00000100:00080000:3.0:1786177262.797497:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:2.0:1786177262.797500:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1786177262.797502:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177262.797503:0:2392:0:(genops.c:1238:class_import_put()) Process entered 02000000:00000001:3.0:1786177262.797505:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000020:00000040:2.0:1786177262.797507:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177262.797510:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1786177262.797512:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:3.0:1786177262.797515:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177262.797516:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa400. 02000000:00000001:2.0:1786177262.797522:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177262.797523:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177262.797525:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324380. 00000100:00000001:3.0:1786177262.797525:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177262.797528:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1786177262.797530:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000008:00000001:3.0:1786177262.797531:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.797533:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177262.797534:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.797539:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1786177262.797540:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786177262.797544:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177262.797545:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177262.797546:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.797548:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177262.797548:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177262.797551:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1786177262.797552:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177262.797553:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.797557:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:3.0:1786177262.797558:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:2.0:1786177262.797559:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177262.797561:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.797563:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00100000:3.0:1786177262.797567:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1786177262.797569:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.797576:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177262.797577:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177262.797579:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177262.797581:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177262.797586:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177262.797592:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595648:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177262.797599:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177262.797602:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177262.797604:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177262.797609:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1786177262.797610:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000010:2.0:1786177262.797613:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:3.0:1786177262.797614:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177262.797617:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177262.797622:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177262.797628:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177262.797631:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11947) 00000400:00000200:2.0:1786177262.797634:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177262.797638:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177262.797642:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177262.797645:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235595648, portal 10 00000100:00000001:2.0:1786177262.797648:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177262.797650:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818944 : -110371428732672 : ffff9b9e26327100) 00000100:00000040:2.0:1786177262.797657:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.797663:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177262.797665:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177262.797667:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177262.797669:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235595648, offset 0 00000400:00000200:2.0:1786177262.797673:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177262.797676:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177262.797680:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177262.797683:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177262.797685:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177262.797687:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177262.797689:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177262.797692:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426720:426720:256:4294967295] 192.168.202.132@tcp LPNI seq info [426682:426682:8:4294967295] 00000400:00000200:2.0:1786177262.797696:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177262.797715:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177262.797719:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177262.797723:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177262.797727:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177262.797729:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177262.797734:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177262.797746:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177262.797750:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177262.797752:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177262.797755:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177262.797758:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177262.797760:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177262.797762:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177262.797764:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177262.797767:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.797778:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177262.797781:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.797788:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.797789:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177262.797792:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177262.797796:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177262.797798:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177262.797800:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306609668 00000100:00000001:2.0:1786177262.797802:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306613504 00000100:00000001:2.0:1786177262.797804:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306609668 00000100:00000001:2.0:1786177262.797806:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306613504 00000100:00000001:2.0:1786177262.797808:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177262.797809:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.797811:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177262.797814:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177262/real 1786177262] req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177262.797820:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177262.797821:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177262.797822:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177262.797824:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177262.797826:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177262.797827:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177262.797830:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177262.797836:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.797860:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177262.797862:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.797863:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177262.797864:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177262.797867:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.797873:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177262.797876:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.797881:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177262.797883:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.797885:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177262.797886:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.797887:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177262.797889:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177262.797891:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177262.797894:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177262.797896:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177262.797899:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177262.797905:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595648:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177262.797909:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177262.797911:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.797917:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177262.797918:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177262.797920:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177262.797921:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177262.797923:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177262.797927:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595776:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177262.797930:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177262.797931:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177262.797933:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177262.797934:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177262.797937:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177262.797939:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177262.797942:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177262.797944:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177262.797947:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177262.797950:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235595776, portal 10 00000100:00000001:2.0:1786177262.797951:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177262.797953:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280816256 : -110371428735360 : ffff9b9e26326680) 00000100:00000040:2.0:1786177262.797958:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.797963:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177262.797965:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177262.797967:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177262.797969:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235595776, offset 0 00000400:00000200:2.0:1786177262.797972:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177262.797975:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177262.797979:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177262.797981:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177262.797984:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177262.797986:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177262.797988:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177262.797991:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426721:426721:256:4294967295] 192.168.202.132@tcp LPNI seq info [426683:426683:8:4294967295] 00000400:00000200:2.0:1786177262.797995:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177262.798000:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177262.798005:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177262.798008:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177262.798011:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177262.798013:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177262.798016:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177262.798017:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177262.798020:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177262.798021:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177262.798023:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177262.798025:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177262.798027:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177262.798028:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177262.798029:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177262.798032:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798036:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177262.798038:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798042:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.798043:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177262.798045:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177262.798047:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177262.798048:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177262.798050:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306609668 00000100:00000001:2.0:1786177262.798051:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306613504 00000100:00000001:2.0:1786177262.798052:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306609668 00000100:00000001:2.0:1786177262.798054:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306613504 00000100:00000001:2.0:1786177262.798055:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177262.798056:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.798057:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177262.798059:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177262/real 1786177262] req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177262.798063:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177262.798064:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177262.798065:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177262.798066:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177262.798067:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177262.798069:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177262.798070:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177262.798074:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798078:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177262.798078:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798079:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177262.798080:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177262.798082:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798085:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177262.798087:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798091:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177262.798092:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798093:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177262.798094:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798096:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177262.798097:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177262.798099:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177262.798102:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177262.798114:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177262.798119:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177262.798132:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595776:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177262.798135:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177262.798138:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798145:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177262.798146:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177262.798148:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177262.798150:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177262.798152:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177262.798154:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595904:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177262.798157:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177262.798159:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177262.798160:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177262.798162:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177262.798165:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177262.798167:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177262.798170:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177262.798173:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177262.798176:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177262.798179:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235595904, portal 4 00000100:00000001:2.0:1786177262.798180:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177262.798181:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280819840 : -110371428731776 : ffff9b9e26327480) 00000100:00000040:2.0:1786177262.798186:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798191:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177262.798193:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177262.798195:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177262.798197:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235595904, offset 0 00000400:00000200:2.0:1786177262.798200:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177262.798204:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177262.798207:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177262.798210:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177262.798212:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177262.798214:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177262.798216:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177262.798218:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426722:426722:256:4294967295] 192.168.202.132@tcp LPNI seq info [426684:426684:8:4294967295] 00000400:00000200:2.0:1786177262.798222:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177262.798226:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177262.798230:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177262.798233:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177262.798245:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177262.798248:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177262.798251:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177262.798253:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177262.798257:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177262.798258:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177262.798260:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177262.798263:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177262.798266:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177262.798267:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177262.798269:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177262.798273:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798279:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177262.798281:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798286:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.798288:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177262.798290:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177262.798293:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177262.798295:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177262.798298:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306609668 00000100:00000001:2.0:1786177262.798300:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306613504 00000100:00000001:2.0:1786177262.798303:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306609668 00000100:00000001:2.0:1786177262.798304:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306613504 00000100:00000001:2.0:1786177262.798306:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177262.798308:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.798310:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177262.798313:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177262/real 1786177262] req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177262.798320:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177262.798321:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177262.798322:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177262.798325:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177262.798326:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177262.798328:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177262.798331:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177262.798338:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798344:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177262.798345:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798347:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177262.798348:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177262.798352:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798376:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177262.798380:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798387:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177262.798390:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798392:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177262.798393:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798395:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177262.798396:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177262.798398:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177262.798402:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177262.798404:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177262.798407:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177262.798413:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235595904:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177262.798417:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798418:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177262.798421:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327100 x1872930235595648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798427:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177262.798429:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177262.798430:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177262.798433:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177262.798434:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177262.798435:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177262.798437:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177262.798439:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab800. 02000000:00000001:2.0:1786177262.798441:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177262.798442:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177262.798443:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327100. 00000100:00000001:2.0:1786177262.798445:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177262.798446:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798448:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177262.798451:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326680 x1872930235595776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798462:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177262.798463:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177262.798464:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177262.798466:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177262.798468:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177262.798469:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177262.798471:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177262.798472:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8c00. 02000000:00000001:2.0:1786177262.798474:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177262.798475:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177262.798477:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326680. 00000100:00000001:2.0:1786177262.798478:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177262.798480:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798481:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177262.798484:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327480 x1872930235595904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798490:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177262.798491:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177262.798492:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177262.798494:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177262.798495:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177262.798496:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177262.798497:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177262.798498:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8400. 02000000:00000001:2.0:1786177262.798499:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177262.798499:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177262.798500:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327480. 00000100:00000001:2.0:1786177262.798501:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177262.798502:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798503:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798505:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177262.798506:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.798507:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177262.798508:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177262.798509:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177262.798511:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798516:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177262.798517:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177262.798519:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177262.798521:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177262.798523:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177262.798526:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596032:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177262.798529:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177262.798531:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177262.798532:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177262.798534:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177262.798537:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177262.798539:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177262.798542:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177262.798544:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177262.798547:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177262.798550:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235596032, portal 4 00000100:00000001:2.0:1786177262.798552:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177262.798553:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280814464 : -110371428737152 : ffff9b9e26325f80) 00000100:00000040:2.0:1786177262.798559:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798565:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177262.798567:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177262.798569:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177262.798571:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235596032, offset 0 00000400:00000200:2.0:1786177262.798575:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177262.798578:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177262.798581:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177262.798584:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177262.798586:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177262.798588:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177262.798590:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177262.798592:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426723:426723:256:4294967295] 192.168.202.132@tcp LPNI seq info [426685:426685:8:4294967295] 00000400:00000200:2.0:1786177262.798594:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177262.798598:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177262.798601:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177262.798603:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177262.798605:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177262.798607:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177262.798610:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177262.798611:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177262.798614:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177262.798615:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177262.798617:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177262.798619:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177262.798620:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177262.798621:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177262.798623:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177262.798625:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798629:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177262.798631:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177287 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798635:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.798636:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177262.798637:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177262.798639:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177262.798641:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177262.798642:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306609671 00000100:00000001:2.0:1786177262.798644:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306613504 00000100:00000001:2.0:1786177262.798645:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306609671 00000100:00000001:2.0:1786177262.798646:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306613504 00000100:00000001:2.0:1786177262.798647:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177262.798649:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.798650:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177262.798652:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177262/real 1786177262] req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177262.798656:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177262.798657:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177262.798658:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177262.798659:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177262.798660:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177262.798661:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177262.798663:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177262.798667:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798671:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177262.798671:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798672:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177262.798673:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177262.798675:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798679:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177262.798682:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798688:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177262.798690:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798692:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177262.798693:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798695:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177262.798696:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177262.798709:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177262.798713:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177262.798715:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177262.798718:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177262.798726:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596032:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177262.798729:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798731:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177262.798734:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325f80 x1872930235596032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177287 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177262.798739:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177262.798741:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177262.798743:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177262.798745:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177262.798746:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177262.798747:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177262.798749:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177262.798750:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:2.0:1786177262.798752:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177262.798753:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177262.798754:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325f80. 00000100:00000001:2.0:1786177262.798757:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177262.798758:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798760:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177262.798763:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177262.798764:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177262.798766:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177262.798767:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177263.308525:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177263.308537:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177263.308541:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177263.308549:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177263.308555:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177263.308561:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177263.308565:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177263.308569:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177263.308573:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177263.692504:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177263.692513:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177263.692515:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177263.692519:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177263.692521:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177263.692525:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1786177263.692529:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177263.692531:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177263.692535:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177263.692537:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177263.692542:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177263.692542:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177263.692548:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177263.692549:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177263.692552:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177263.692555:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177263.692558:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177263.692561:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177263.692562:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177263.692568:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177263.692571:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177263.692572:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177263.692574:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177263.692576:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177263.692577:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177263.692579:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177263.692581:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177263.695761:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177263.695766:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177263.695771:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177264.332464:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177264.332474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177264.332476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177264.332483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177264.332489:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177264.332492:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177264.332496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177264.332498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177264.332502:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177264.716489:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177264.716493:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177264.716497:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177264.716498:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177264.716501:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177264.716504:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177264.716505:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177264.716505:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177264.716509:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177264.716510:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177264.716510:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177264.716514:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177264.716516:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177264.716520:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177264.716523:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177264.716576:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177264.716581:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177264.716585:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177265.356475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177265.356482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177265.356484:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177265.356490:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177265.356494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177265.356497:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177265.356500:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177265.356502:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177265.356505:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177265.740522:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177265.740525:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177265.740534:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177265.740534:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177265.740538:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177265.740546:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177265.740547:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177265.740552:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177265.740555:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177265.740559:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177265.740561:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177265.740564:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177265.740567:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177265.740572:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177265.740576:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177265.740634:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177265.740639:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177265.740643:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177266.380449:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177266.380456:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177266.380459:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177266.380464:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177266.380469:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177266.380473:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177266.380475:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177266.380478:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177266.380481:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177266.700469:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306613504, at 4306613568 00000100:00000040:1.0:1786177266.700469:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306613504, at 4306613568 00000100:00000010:1.0:1786177266.700482:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c060. 00000100:00000010:0.0:1786177266.700485:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290ccc0. 00000100:00000001:1.0:1786177266.764456:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177266.764458:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177266.764462:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177266.764465:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177266.764468:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177266.764474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177266.764474:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177266.764479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177266.764479:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177266.764485:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177266.764485:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177266.764486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177266.764490:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177266.764494:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177266.764496:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177266.764532:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177266.764535:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177266.764538:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177267.404473:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177267.404482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177267.404485:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177267.404491:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177267.404496:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177267.404499:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177267.404503:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177267.404505:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177267.404508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177267.788484:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177267.788494:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177267.788497:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177267.788497:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177267.788502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177267.788505:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177267.788508:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177267.788512:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177267.788512:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177267.788518:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177267.788519:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177267.788521:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177267.788524:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177267.788528:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177267.788532:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177267.788710:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177267.788714:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177267.788718:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786177267.916548:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177267.916557:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786177267.916560:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177267.916563:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177267.916567:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177267.916574:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11942 00000400:00000200:0.0:1786177267.916583:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177267.916590:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177267.916593:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177267.916596:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.916599:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177267.916601:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.916604:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177267.916608:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177267.916610:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786177267.916612:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177267.916615:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786177267.916618:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786177267.916620:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786177267.916622:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.916624:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177267.916626:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.916628:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177267.916630:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786177267.916633:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177267.916635:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177267.916638:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177267.916641:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177267.916644:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.916646:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177267.916663:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177267.916672:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786177267.916676:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786177267.916679:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177267.916681:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.916684:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177267.916686:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177267.916689:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177267.916696:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786177267.916702:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177267.916711:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177267.916719:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177267.916723:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177267.916727:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177267.916748:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177267.916751:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177267.916754:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786177267.916756:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177267.916757:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177267.916760:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177267.916762:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11942 00000400:00000200:0.0:1786177267.916765:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177267.916769:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177267.916771:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177267.916775:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.916777:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177267.916780:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.916782:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177267.916782:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1786177267.916787:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177267.916787:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1786177267.916789:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000100:00000001:2.0:1786177267.916791:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1786177267.916792:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177267.916796:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786177267.916800:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177267.916801:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1786177267.916802:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786177267.916810:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.916816:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177267.916818:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177267.916820:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177267.916823:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177267.916825:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177267.916828:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177267.916832:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235596160:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177267.916836:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177267.916840:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177267.916842:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177267.916845:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177267.916850:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177267.916852:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177267.916857:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177267.916860:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177267.916877:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177267.916883:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235596160, portal 25 00000100:00000001:2.0:1786177267.916885:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177267.916887:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794208768 : -110371915342848 : ffff9b9e09315c00) 00000100:00000040:2.0:1786177267.916893:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.916906:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177267.916909:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177267.916911:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177267.916914:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235596160, offset 0 00000400:00000200:2.0:1786177267.916920:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177267.916927:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177267.916932:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177267.916936:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177267.916940:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177267.916943:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177267.916945:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177267.916949:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426724:426724:256:4294967295] 192.168.202.132@tcp LPNI seq info [426686:426686:8:4294967295] 00000400:00000200:2.0:1786177267.916960:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177267.917251:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177267.917258:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177267.917263:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177267.917270:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177267.917273:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786177267.917278:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177267.917281:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786177267.917286:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177267.917289:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177267.917293:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177267.917297:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177267.917302:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177267.917304:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177267.917308:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177267.917312:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.917322:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177267.917325:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.917338:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.917340:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177267.917345:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177267.917349:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:0.0:1786177267.917353:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177267.917353:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306618624 00000100:00000001:0.0:1786177267.917388:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177267.917392:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786177267.917395:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177267.917398:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177267.917400:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177267.917405:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177267.917408:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.917409:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177267.917413:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786177267.917416:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.917417:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177267.917421:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786177267.917423:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786177267.917426:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000010:2.0:1786177267.917427:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c8a0. 02000000:00000001:0.0:1786177267.917428:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.917432:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177267.917434:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177267.917436:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.917437:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 02000000:00000010:0.0:1786177267.917441:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786177267.917446:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.917446:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306618625 00000100:00100000:0.0:1786177267.917453:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:2.0:1786177267.917453:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cc60. 00000100:00000001:2.0:1786177267.917458:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177267.917463:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177267.917466:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177267.917467:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.917471:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.917472:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1786177267.917477:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177267.917479:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177267/real 1786177267] req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177267.917498:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1786177267.917501:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1786177267.917502:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:2.0:1786177267.917505:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1786177267.917507:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:2.0:1786177267.917509:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000100:00000001:0.0:1786177267.917510:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177267.917514:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1786177267.917515:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000100:00000001:0.0:1786177267.917519:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:2.0:1786177267.917519:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177267.917524:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1786177267.917526:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11942 00000100:00000200:2.0:1786177267.917531:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1786177267.917535:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177267.917542:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177267.917546:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000200:2.0:1786177267.917548:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786177267.917552:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.917557:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177267.917562:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177267.917564:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177267.917566:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177267.917567:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1786177267.917571:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:2.0:1786177267.917572:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:0.0:1786177267.917575:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000100:00000001:2.0:1786177267.917575:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1786177267.917579:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177267.917583:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000100:00100000:2.0:1786177267.917584:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1786177267.917586:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786177267.917592:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786177267.917599:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177267.917602:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177267.917603:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177267.917609:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:2.0:1786177267.917610:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177267.917614:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177267.917620:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786177267.917626:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177267.917626:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177267.917632:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.917632:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1786177267.917637:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1786177267.917637:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177267.917640:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1786177267.917642:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177267.917645:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.917647:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177267.917660:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1786177267.917662:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00000001:0.0:1786177267.917663:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.917666:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177267.917667:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177267.917671:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000010:0.0:1786177267.917674:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000100:00080000:2.0:1786177267.917678:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000020:00000040:0.0:1786177267.917679:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000001:2.0:1786177267.917684:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1786177267.917685:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177267.917687:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.917691:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:2.0:1786177267.917692:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786177267.917693:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177267.917696:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177267.917703:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00100000:2.0:1786177267.917706:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235596160:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177267.917708:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.917716:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177267.917718:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177267.917719:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177267.917725:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09315c00 x1872930235596160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177267.917733:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177267.917737:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.917739:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177267.917742:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:0.0:1786177267.917744:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1786177267.917745:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177267.917750:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177267.917752:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177267.917755:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000001:0.0:1786177267.917758:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177267.917759:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00080000:0.0:1786177267.917762:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:2.0:1786177267.917763:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1c00. 00000100:00080000:0.0:1786177267.917769:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:2.0:1786177267.917770:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1786177267.917772:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1786177267.917773:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:0.0:1786177267.917776:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:2.0:1786177267.917777:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09315c00. 00000100:00000001:0.0:1786177267.917783:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1786177267.917783:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177267.917786:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1786177267.917787:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11942 00000100:00000001:2.0:1786177267.917789:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1786177267.917794:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177267.917797:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:2.0:1786177267.917798:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1786177267.917800:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1786177267.917802:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177267.917805:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177267.917807:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177267.917810:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177267.917811:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1786177267.917812:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177267.917815:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:2.0:1786177267.917819:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.917827:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177267.917828:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177267.917830:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177267.917831:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177267.917833:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177267.917836:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596288:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177267.917839:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177267.917841:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177267.917842:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177267.917844:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177267.917847:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177267.917848:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177267.917852:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177267.917854:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177267.917858:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177267.917860:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235596288, portal 10 00000100:00000001:2.0:1786177267.917862:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177267.917863:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794212352 : -110371915339264 : ffff9b9e09316a00) 00000100:00000040:2.0:1786177267.917869:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.917875:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177267.917877:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177267.917880:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177267.917882:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235596288, offset 0 00000400:00000200:2.0:1786177267.917885:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177267.917889:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177267.917892:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177267.917895:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177267.917898:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177267.917900:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177267.917902:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177267.917905:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426725:426725:256:4294967295] 192.168.202.132@tcp LPNI seq info [426687:426687:8:4294967295] 00000400:00000200:2.0:1786177267.917909:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177267.917915:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177267.917919:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177267.917922:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177267.917925:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177267.917928:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786177267.917932:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177267.917935:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786177267.917938:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177267.917940:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177267.917942:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177267.917946:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177267.917948:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177267.918059:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177267.918063:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177267.918067:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918075:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177267.918078:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918084:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.918086:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177267.918089:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177267.918093:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177267.918096:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177267.918098:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306614788 00000100:00000001:2.0:1786177267.918100:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306618624 00000100:00000001:2.0:1786177267.918102:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306614788 00000100:00000001:2.0:1786177267.918104:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306618625 00000100:00000001:2.0:1786177267.918106:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177267.918108:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.918110:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177267.918113:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177267/real 1786177267] req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177267.918120:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177267.918122:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177267.918123:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177267.918125:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177267.918127:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177267.918129:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177267.918132:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177267.918140:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918146:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177267.918147:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.918148:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177267.918150:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177267.918153:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918159:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177267.918162:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918169:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177267.918172:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.918173:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177267.918174:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.918176:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177267.918178:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177267.918180:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177267.918183:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177267.918185:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177267.918188:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177267.918196:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596288:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177267.918199:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177267.918202:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918208:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177267.918209:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177267.918211:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177267.918212:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177267.918214:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177267.918217:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596416:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177267.918220:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177267.918222:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177267.918223:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177267.918225:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177267.918228:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177267.918229:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177267.918233:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177267.918235:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177267.918237:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177267.918240:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235596416, portal 10 00000100:00000001:2.0:1786177267.918242:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177267.918243:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215936 : -110371915335680 : ffff9b9e09317800) 00000100:00000040:2.0:1786177267.918247:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918253:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177267.918256:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177267.918257:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177267.918260:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235596416, offset 0 00000400:00000200:2.0:1786177267.918263:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177267.918266:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177267.918269:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177267.918271:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177267.918273:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177267.918276:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177267.918278:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177267.918281:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426726:426726:256:4294967295] 192.168.202.132@tcp LPNI seq info [426688:426688:8:4294967295] 00000400:00000200:2.0:1786177267.918285:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177267.918290:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177267.918294:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177267.918297:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177267.918301:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177267.918303:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786177267.918307:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177267.918310:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786177267.918313:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177267.918315:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177267.918318:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177267.918321:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177267.918323:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177267.918325:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177267.918327:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177267.918330:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918337:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177267.918340:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918346:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.918348:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177267.918350:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177267.918353:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177267.918355:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177267.918372:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306614788 00000100:00000001:2.0:1786177267.918374:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306618624 00000100:00000001:2.0:1786177267.918376:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306614788 00000100:00000001:2.0:1786177267.918378:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306618625 00000100:00000001:2.0:1786177267.918380:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177267.918382:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.918384:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177267.918387:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177267/real 1786177267] req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177267.918394:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177267.918396:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177267.918397:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177267.918400:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177267.918401:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177267.918403:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177267.918406:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177267.918414:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918420:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177267.918422:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.918423:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177267.918424:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177267.918428:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918433:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177267.918437:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918443:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177267.918445:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.918446:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177267.918448:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.918449:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177267.918451:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177267.918453:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177267.918455:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177267.918457:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177267.918461:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177267.918468:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596416:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177267.918471:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.918473:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177267.918475:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316a00 x1872930235596288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918481:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177267.918483:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177267.918485:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177267.918487:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177267.918488:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177267.918489:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177267.918491:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177267.918493:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:2.0:1786177267.918495:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177267.918496:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177267.918498:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316a00. 00000100:00000001:2.0:1786177267.918500:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177267.918501:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.918503:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177267.918505:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317800 x1872930235596416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.918512:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177267.918513:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177267.918514:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177267.918516:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177267.918517:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177267.918518:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177267.918519:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177267.918520:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:2.0:1786177267.918522:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177267.918523:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177267.918524:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317800. 00000100:00000001:2.0:1786177267.918526:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177267.918527:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.918529:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.918532:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177267.918533:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.918535:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177267.918536:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.918943:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177267.918946:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177267.918948:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786177267.918949:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177267.918952:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786177267.918953:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786177267.918956:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786177267.918959:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.918961:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177267.918963:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.918965:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177267.918968:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786177267.918970:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177267.918973:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177267.918975:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177267.918978:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177267.918980:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.918982:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177267.918984:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786177267.918986:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.918988:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177267.918992:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786177267.918994:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177267.918997:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177267.918999:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.919001:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177267.919003:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177267.919004:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177267.919009:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786177267.919012:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177267.919018:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177267.919025:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177267.919028:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177267.919032:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177267.919052:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177267.919055:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177267.919057:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786177267.919058:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177267.919060:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177267.919062:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177267.919065:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11942 00000400:00000200:0.0:1786177267.919069:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177267.919072:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177267.919074:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177267.919076:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.919078:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177267.919080:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.919082:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177267.919084:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177267.919085:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786177267.919087:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177267.919088:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786177267.919090:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786177267.919092:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786177267.919095:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.919097:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177267.919099:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177267.919101:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177267.919103:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786177267.919105:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177267.919108:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177267.919110:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177267.919113:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177267.919115:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.919117:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177267.919119:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786177267.919121:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.919122:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177267.919127:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786177267.919129:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177267.919133:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177267.919134:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.919136:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177267.919138:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1786177267.919138:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1786177267.919143:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.919143:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177267.919146:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:0.0:1786177267.919149:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000040:2.0:1786177267.919154:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177267.919155:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177267.919164:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177267.919167:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177267.919169:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177267.919174:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177267.919176:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177267.919178:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1786177267.919180:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177267.919183:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177267.919186:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596544:192.168.202.132@tcp:8:kworker.0 00000100:00000040:0.0:1786177267.919191:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919192:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177267.919195:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177267.919198:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177267.919201:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177267.919205:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:0.0:1786177267.919207:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177267.919209:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177267.919214:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11952) 00000400:00000200:2.0:1786177267.919215:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177267.919220:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177267.919224:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177267.919227:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235596544, portal 4 00000100:00000001:2.0:1786177267.919228:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177267.919230:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794210560 : -110371915341056 : ffff9b9e09316300) 00000100:00000040:2.0:1786177267.919236:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919243:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177267.919245:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177267.919247:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177267.919249:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235596544, offset 0 00000400:00000200:2.0:1786177267.919253:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177267.919256:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177267.919259:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177267.919262:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177267.919264:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177267.919266:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177267.919269:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177267.919272:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426727:426727:256:4294967295] 192.168.202.132@tcp LPNI seq info [426689:426689:8:4294967295] 00000400:00000200:2.0:1786177267.919276:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177267.919282:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177267.919286:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177267.919290:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177267.919293:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177267.919296:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786177267.919300:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177267.919303:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786177267.919307:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177267.919308:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177267.919311:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177267.919314:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177267.919317:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177267.919318:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177267.919321:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177267.919324:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919331:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177267.919334:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919340:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.919342:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177267.919344:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177267.919346:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177267.919349:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177267.919351:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306614789 00000100:00000001:2.0:1786177267.919353:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306618624 00000100:00000001:2.0:1786177267.919355:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306614789 00000100:00000001:2.0:1786177267.919373:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306618625 00000100:00000001:2.0:1786177267.919375:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177267.919378:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.919379:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177267.919383:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177267/real 1786177267] req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177267.919390:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177267.919392:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177267.919393:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177267.919396:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177267.919397:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177267.919399:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177267.919402:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177267.919410:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919416:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177267.919418:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919419:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177267.919421:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177267.919423:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919429:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177267.919433:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919439:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177267.919441:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919443:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177267.919444:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919445:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177267.919447:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177267.919449:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177267.919452:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177267.919454:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177267.919458:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177267.919465:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596544:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177267.919468:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919470:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177267.919472:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316300 x1872930235596544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919478:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177267.919480:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177267.919481:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177267.919483:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177267.919485:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177267.919486:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177267.919488:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177267.919489:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:2.0:1786177267.919491:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177267.919493:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177267.919494:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316300. 00000100:00000001:2.0:1786177267.919496:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177267.919497:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919499:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919502:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177267.919503:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.919505:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177267.919507:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177267.919508:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177267.919511:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919517:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177267.919519:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177267.919521:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177267.919522:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177267.919524:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177267.919526:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596672:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177267.919529:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177267.919531:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177267.919532:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177267.919534:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177267.919536:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177267.919538:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177267.919541:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177267.919543:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177267.919546:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177267.919548:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235596672, portal 4 00000100:00000001:2.0:1786177267.919550:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177267.919551:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794202496 : -110371915349120 : ffff9b9e09314380) 00000100:00000040:2.0:1786177267.919556:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919562:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177267.919564:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177267.919566:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177267.919568:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235596672, offset 0 00000400:00000200:2.0:1786177267.919571:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177267.919574:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177267.919577:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177267.919580:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177267.919582:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177267.919584:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177267.919586:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177267.919589:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426728:426728:256:4294967295] 192.168.202.132@tcp LPNI seq info [426690:426690:8:4294967295] 00000400:00000200:2.0:1786177267.919593:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177267.919599:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177267.919603:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177267.919606:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177267.919610:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177267.919612:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786177267.919616:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177267.919618:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786177267.919622:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177267.919623:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177267.919626:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177267.919629:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177267.919631:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177267.919633:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177267.919635:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177267.919638:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919644:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177267.919647:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177292 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919670:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.919672:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177267.919674:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177267.919686:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177267.919689:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177267.919691:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306614792 00000100:00000001:2.0:1786177267.919695:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306618624 00000100:00000001:2.0:1786177267.919697:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306614792 00000100:00000001:2.0:1786177267.919699:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306618625 00000100:00000001:2.0:1786177267.919701:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177267.919703:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.919705:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177267.919708:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177267/real 1786177267] req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177267.919716:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177267.919717:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177267.919719:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177267.919721:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177267.919722:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177267.919724:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177267.919727:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177267.919735:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919740:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177267.919742:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919743:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177267.919745:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177267.919748:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919754:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177267.919758:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919764:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177267.919766:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919767:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177267.919769:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919770:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177267.919772:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177267.919774:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177267.919777:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177267.919779:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177267.919782:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177267.919790:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596672:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177267.919793:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919795:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177267.919797:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314380 x1872930235596672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177292 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177267.919803:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177267.919804:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177267.919806:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:2.0:1786177267.919808:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177267.919809:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177267.919811:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177267.919812:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177267.919814:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:2.0:1786177267.919816:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177267.919817:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177267.919818:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314380. 00000100:00000001:2.0:1786177267.919821:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177267.919822:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919824:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177267.919826:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177267.919828:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177267.919829:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177267.919831:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177268.428698:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177268.428705:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177268.428707:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177268.428713:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177268.428716:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177268.428719:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177268.428722:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177268.428724:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177268.428727:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177268.812460:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177268.812467:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177268.812474:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177268.812474:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177268.812475:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177268.812478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177268.812478:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177268.812480:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177268.812480:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177268.812484:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177268.812487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177268.812490:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177268.812498:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177268.812501:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177268.812504:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177268.812541:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177268.812545:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177268.812549:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177269.452449:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177269.452457:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177269.452459:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177269.452465:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177269.452469:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177269.452473:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177269.452476:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177269.452478:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177269.452481:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177269.836461:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177269.836467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177269.836469:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177269.836473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177269.836474:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177269.836482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177269.836482:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177269.836485:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177269.836488:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177269.836491:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177269.836493:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177269.836495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177269.836499:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177269.836502:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177269.836506:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177269.836597:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177269.836600:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177269.836604:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177270.476467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177270.476474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177270.476477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177270.476482:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177270.476486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177270.476490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177270.476498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177270.476501:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177270.476505:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177270.860488:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177270.860495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177270.860500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177270.860502:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177270.860504:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177270.860507:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177270.860510:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177270.860512:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177270.860514:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177270.860517:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177270.860520:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177270.860521:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177270.860593:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177270.860598:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177270.860600:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177270.860603:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177270.860607:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177270.860611:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177270.860613:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177270.860615:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177270.860617:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177270.860777:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177270.860780:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177270.860782:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177271.500587:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177271.500595:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177271.500597:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177271.500603:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177271.500607:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177271.500610:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177271.500612:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177271.500622:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177271.500626:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177271.820469:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306618625, at 4306618688 00000100:00000010:1.0:1786177271.820485:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cc60. 00000100:00000040:0.0:1786177271.822711:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306618624, at 4306618690 00000100:00000010:0.0:1786177271.822723:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c8a0. 00000100:00000001:1.0:1786177271.884475:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177271.884479:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177271.884484:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177271.884484:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177271.884488:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177271.884492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177271.884494:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177271.884495:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177271.884500:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177271.884502:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177271.884505:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177271.884506:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177271.884511:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177271.884516:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177271.884519:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177271.884640:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177271.884644:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177271.884647:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177272.524456:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177272.524464:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177272.524466:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177272.524472:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177272.524476:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177272.524480:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177272.524483:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177272.524485:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177272.524489:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177272.908478:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177272.908494:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177272.908500:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177272.908501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177272.908505:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177272.908506:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177272.908663:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177272.908668:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177272.908670:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177272.908673:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177272.908677:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177272.908680:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177272.908683:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177272.908685:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177272.908688:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177272.908751:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177272.908754:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177272.908757:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177273.036705:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177273.036716:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177273.036718:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177273.036722:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177273.036727:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177273.036734:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11947 00000400:00000200:2.0:1786177273.036741:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177273.036748:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177273.036752:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177273.036755:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.036759:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177273.036761:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.036763:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177273.036768:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177273.036770:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177273.036785:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177273.036788:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177273.036791:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177273.036793:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177273.036797:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.036798:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177273.036800:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.036802:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177273.036805:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177273.036808:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177273.036812:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177273.036815:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177273.036819:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177273.036822:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.036824:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177273.036826:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177273.036835:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735180. 00000020:00000040:2.0:1786177273.036839:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177273.036843:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177273.036844:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.036847:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177273.036850:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177273.036854:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177273.036862:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:2.0:1786177273.036868:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177273.036877:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177273.036887:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177273.036891:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177273.036897:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ba0f078] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177273.036925:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177273.036928:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177273.036932:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177273.036933:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177273.036935:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177273.036938:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177273.036941:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11947 00000400:00000200:2.0:1786177273.036945:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177273.036948:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177273.036950:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177273.036953:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.036955:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177273.036957:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.036959:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177273.036961:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177273.036963:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177273.036964:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177273.036966:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177273.036968:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177273.036970:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177273.036973:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.036975:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177273.036977:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.036979:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177273.036981:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177273.036984:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177273.036986:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177273.036988:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177273.036992:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177273.036994:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.036996:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177273.037000:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177273.037014:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.037016:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177273.037020:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734e00. 00000020:00000040:2.0:1786177273.037022:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177273.037025:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177273.037026:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.037028:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177273.037029:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177273.037031:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177273.037034:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:2.0:1786177273.037038:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177273.037044:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177273.037052:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177273.037054:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177273.037057:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001f62212] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177273.037065:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177273.037067:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177273.037070:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177273.037072:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177273.037073:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177273.037076:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177273.037078:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11947 00000400:00000200:2.0:1786177273.037082:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177273.037085:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177273.037087:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177273.037090:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037092:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177273.037094:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037095:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177273.037098:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177273.037099:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177273.037100:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177273.037102:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177273.037104:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177273.037106:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177273.037109:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037111:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177273.037113:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037115:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177273.037117:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177273.037120:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037122:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177273.037124:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177273.037127:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177273.037129:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.037131:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177273.037132:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177273.037134:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.037136:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177273.037141:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000020:00000040:2.0:1786177273.037143:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177273.037145:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177273.037147:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.037148:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177273.037149:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177273.037151:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177273.037155:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177273.037158:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177273.037163:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177273.037170:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177273.037172:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177273.037175:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177273.037183:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177273.037185:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177273.037188:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177273.037190:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177273.037191:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177273.037195:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177273.037197:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11947 00000400:00000200:2.0:1786177273.037200:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177273.037203:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177273.037205:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177273.037208:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037210:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177273.037212:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037214:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177273.037216:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177273.037217:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177273.037218:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177273.037220:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177273.037222:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177273.037225:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177273.037227:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037229:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177273.037231:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037233:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177273.037235:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177273.037238:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037240:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177273.037242:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177273.037244:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177273.037246:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.037248:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177273.037250:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177273.037253:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.037254:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177273.037257:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000020:00000040:2.0:1786177273.037260:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177273.037262:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177273.037264:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.037265:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177273.037266:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177273.037268:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177273.037271:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786177273.037274:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177273.037279:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177273.037286:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177273.037288:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177273.037292:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177273.037299:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177273.037301:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177273.037304:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177273.037317:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177273.037318:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177273.037321:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177273.037324:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11947 00000400:00000200:2.0:1786177273.037327:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177273.037330:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177273.037332:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177273.037334:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037336:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177273.037338:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037340:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177273.037342:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177273.037343:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177273.037345:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177273.037347:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177273.037349:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177273.037350:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177273.037353:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.037354:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177273.037582:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177273.037587:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177273.037589:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177273.037596:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.037614:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177273.037617:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177273.037620:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177273.037624:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177273.037627:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177273.037630:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235596800:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177273.037635:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177273.037639:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177273.037641:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177273.037644:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177273.037649:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ad000. 02000000:00000001:1.0:1786177273.037652:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177273.037657:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177273.037662:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177273.037669:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177273.037676:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235596800, portal 25 00000100:00000001:1.0:1786177273.037678:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177273.037680:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620096 : -110371239931520 : ffff9b9e31735180) 00000100:00000040:1.0:1786177273.037686:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.037693:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177273.037695:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177273.037698:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177273.037701:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235596800, offset 0 00000400:00000200:1.0:1786177273.037708:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177273.037715:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177273.037721:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177273.037726:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177273.037730:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177273.037734:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177273.037737:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177273.037740:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426729:426729:256:4294967295] 192.168.202.132@tcp LPNI seq info [426691:426691:8:4294967295] 00000400:00000200:1.0:1786177273.037745:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177273.037755:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177273.037761:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177273.037998:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177273.038065:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177273.038069:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177273.038074:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177273.038078:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177273.038082:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177273.038085:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177273.038090:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177273.038095:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177273.038101:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177273.038104:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177273.038107:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177273.038112:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.038122:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177273.038126:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.038133:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177273.038135:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177273.038142:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177273.038146:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177273.038149:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306623745 00000100:00000010:1.0:1786177273.038154:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62ea0. 00000100:00000001:1.0:1786177273.038160:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177273.038167:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306623745 00000100:00000010:1.0:1786177273.038169:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c624e0. 00000100:00000001:1.0:1786177273.038172:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177273.038176:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177273.038179:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177273.038181:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177273.038186:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177273/real 1786177273] req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177273.038201:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177273.038203:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177273.038205:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177273.038207:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177273.038209:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177273.038211:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177273.038215:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177273.038225:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.038232:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177273.038234:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.038236:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177273.038238:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177273.038242:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.038249:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177273.038253:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.038260:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177273.038263:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.038264:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177273.038265:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.038268:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177273.038270:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177273.038273:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177273.038277:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177273.038279:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177273.038284:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177273.038292:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235596800:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177273.038297:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177273.038301:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.038308:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177273.038309:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177273.038311:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177273.038313:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177273.038315:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177273.038318:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596928:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177273.038321:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177273.038323:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177273.038325:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177273.038326:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177273.038329:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261af400. 02000000:00000001:1.0:1786177273.038331:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177273.038335:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177273.038337:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177273.038341:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177273.038344:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235596928, portal 10 00000100:00000001:1.0:1786177273.038346:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177273.038347:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469619200 : -110371239932416 : ffff9b9e31734e00) 00000100:00000040:1.0:1786177273.038354:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.038970:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177273.038974:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177273.038977:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177273.038981:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235596928, offset 0 00000400:00000200:1.0:1786177273.038986:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177273.038991:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177273.038995:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000040:2.0:1786177273.039279:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177273.039283:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177273.039286:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177273.039290:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177273.039294:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177273.039296:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177273.039300:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177273.039302:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.039476:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177273.039480:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177273.039483:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.039485:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177273.039490:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000020:00000040:2.0:1786177273.039493:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177273.039496:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177273.039498:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177273.039501:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177273.039502:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177273.039505:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177273.039510:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000001:2.0:1786177273.039514:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177273.039520:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177273.039528:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177273.039532:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177273.039536:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177273.039545:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177273.039550:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11957) 00000400:00000200:1.0:1786177273.039811:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177273.039818:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177273.039821:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177273.039824:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177273.039827:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426730:426730:256:4294967295] 192.168.202.132@tcp LPNI seq info [426692:426692:8:4294967295] 00000400:00000200:1.0:1786177273.039833:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177273.039841:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177273.039847:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177273.039852:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177273.039857:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177273.039861:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177273.039865:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177273.039869:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177273.039873:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177273.039875:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177273.039879:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177273.039882:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177273.039888:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177273.039890:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177273.039893:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177273.039897:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.039905:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177273.039909:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.039915:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177273.039917:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177273.039921:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177273.039925:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177273.039929:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177273.039932:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306619910 00000100:00000001:1.0:1786177273.039935:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306623745 00000100:00000001:1.0:1786177273.039938:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306619910 00000100:00000001:1.0:1786177273.039939:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306623745 00000100:00000001:1.0:1786177273.039941:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177273.039944:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177273.039945:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177273.039950:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177273/real 1786177273] req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177273.039958:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177273.039960:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177273.039962:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177273.039964:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177273.039966:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177273.039968:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177273.039972:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177273.039981:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.039987:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177273.039989:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.039991:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177273.039993:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177273.039996:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.040479:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177273.040485:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.040493:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177273.040496:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.040498:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177273.040499:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.040501:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177273.040504:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177273.040507:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177273.040511:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177273.040513:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177273.040517:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177273.040526:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235596928:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177273.040530:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177273.040534:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.040541:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177273.040542:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177273.040545:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177273.040547:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177273.040550:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177273.040553:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597056:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177273.040556:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177273.040559:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177273.040561:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177273.040563:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177273.040567:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261adc00. 02000000:00000001:1.0:1786177273.040569:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177273.040573:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177273.040576:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177273.040581:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177273.040585:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235597056, portal 10 00000100:00000001:1.0:1786177273.040588:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177273.040589:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469618304 : -110371239933312 : ffff9b9e31734a80) 00000100:00000040:1.0:1786177273.040595:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.041112:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177273.041116:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177273.041120:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177273.041123:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235597056, offset 0 00000400:00000200:1.0:1786177273.041128:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177273.041133:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177273.041139:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177273.041143:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177273.041145:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177273.041148:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177273.041151:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177273.041154:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426731:426731:256:4294967295] 192.168.202.132@tcp LPNI seq info [426693:426693:8:4294967295] 00000400:00000200:1.0:1786177273.041158:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177273.041165:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177273.041170:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177273.041174:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177273.041178:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177273.041181:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1786177273.041185:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177273.041188:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1786177273.041192:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177273.041194:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177273.041198:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177273.041202:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177273.041205:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:1.0:1786177273.041207:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:1.0:1786177273.041210:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177273.041214:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.041229:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177273.041233:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.041239:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177273.041241:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177273.041244:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177273.041248:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177273.041251:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177273.041253:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306619911 00000100:00000001:1.0:1786177273.041256:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306623745 00000100:00000001:1.0:1786177273.041258:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306619911 00000100:00000001:1.0:1786177273.041260:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306623745 00000100:00000001:1.0:1786177273.041262:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177273.041264:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177273.041266:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177273.041270:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177273/real 1786177273] req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177273.041278:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177273.041280:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177273.041282:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:1.0:1786177273.041284:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:1.0:1786177273.041286:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:1.0:1786177273.041288:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177273.041291:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177273.041300:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.043865:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177273.043870:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.043873:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177273.043875:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177273.043884:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.043894:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177273.043899:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.043906:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177273.043910:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.043911:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177273.043913:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.043915:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177273.043918:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177273.043921:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177273.043925:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177273.043928:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177273.043932:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177273.043940:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597056:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177273.043946:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177273.043950:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.043957:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177273.043958:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177273.043962:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177273.043964:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177273.043967:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177273.043970:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597184:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177273.043974:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177273.043977:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177273.043979:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177273.043982:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177273.043988:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261acc00. 02000000:00000001:1.0:1786177273.043990:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177273.043995:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177273.043999:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e06842ec8. 00000400:00000010:1.0:1786177273.044249:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027eee0. 00000100:00000200:1.0:1786177273.044257:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235597184, portal 4 00000100:00000001:1.0:1786177273.044259:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177273.044261:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469628160 : -110371239923456 : ffff9b9e31737100) 00000100:00000040:1.0:1786177273.044270:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177273.044276:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177273.044279:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177273.044282:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e550. 00000100:00000200:1.0:1786177273.044285:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235597184, offset 0 00000400:00000200:1.0:1786177273.044291:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177273.044298:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177273.044466:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177273.044470:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177273.044474:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177273.044477:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177273.044481:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177273.044484:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426732:426732:256:4294967295] 192.168.202.132@tcp LPNI seq info [426694:426694:8:4294967295] 00000400:00000200:1.0:1786177273.044489:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177273.044497:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177273.044503:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177273.044617:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177273.044625:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177273.044628:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1786177273.044634:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177273.044637:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1786177273.044641:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177273.044643:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177273.044648:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177273.044652:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177273.044657:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e550 00000400:00000010:0.0:1786177273.044660:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e550. 00000100:00000001:0.0:1786177273.044664:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177273.044669:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.044677:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177273.044681:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.044688:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177273.044691:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177273.044697:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177273.044700:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177273.044704:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177273.044707:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306619915 00000100:00000001:0.0:1786177273.044711:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306623745 00000100:00000001:0.0:1786177273.044713:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306619915 00000100:00000001:0.0:1786177273.044715:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306623745 00000100:00000001:0.0:1786177273.044717:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177273.044720:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177273.044722:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177273.044726:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177273/real 1786177273] req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177273.044735:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177273.044737:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177273.044739:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e06842ec8. 00000400:00000200:0.0:1786177273.044741:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027eee0 00000400:00000010:0.0:1786177273.044743:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027eee0. 00000100:00000001:0.0:1786177273.044746:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177273.044749:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177273.044758:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.044764:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177273.044765:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.044767:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177273.044769:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177273.044841:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.044848:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177273.044853:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.044860:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177273.044863:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.044864:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177273.044866:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.044868:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177273.044869:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177273.044872:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177273.044876:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177273.044878:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177273.044882:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177273.044890:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597184:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177273.044895:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.044898:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177273.044902:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735180 x1872930235596800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.044909:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177273.044911:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177273.044914:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:0.0:1786177273.044918:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177273.044920:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177273.044922:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177273.044924:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177273.044927:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:0.0:1786177273.044930:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177273.044932:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177273.044933:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735180. 00000100:00000001:0.0:1786177273.044937:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177273.044938:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.044940:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177273.044944:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734e00 x1872930235596928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.044951:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177273.044952:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177273.044954:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261af400. 02000000:00000001:0.0:1786177273.044956:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177273.044957:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177273.044958:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177273.044960:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177273.044962:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:0.0:1786177273.044964:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177273.044965:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177273.044966:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734e00. 00000100:00000001:0.0:1786177273.044968:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177273.044970:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.044971:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177273.044975:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734a80 x1872930235597056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.044981:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177273.044982:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177273.044984:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261adc00. 02000000:00000001:0.0:1786177273.044986:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177273.044987:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177273.044988:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177273.044990:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177273.044991:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:0.0:1786177273.044993:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177273.044994:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177273.044996:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734a80. 00000100:00000001:0.0:1786177273.044998:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177273.044999:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.045001:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177273.045069:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737100 x1872930235597184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.045082:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177273.045083:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177273.045085:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:0.0:1786177273.045088:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177273.045089:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177273.045090:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177273.045092:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177273.045094:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:0.0:1786177273.045096:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177273.045097:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177273.045098:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737100. 00000100:00000001:0.0:1786177273.045100:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177273.045102:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.045104:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.045111:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177273.045113:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177273.045116:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177273.045117:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177273.045119:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177273.045123:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.045130:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177273.045131:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177273.045133:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177273.045135:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177273.045138:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177273.045141:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597312:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177273.045145:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177273.045147:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177273.045149:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177273.045150:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177273.045154:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177273.045156:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177273.045160:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177273.045164:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177273.045168:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177273.045171:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235597312, portal 4 00000100:00000001:0.0:1786177273.045173:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177273.045175:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469623680 : -110371239927936 : ffff9b9e31735f80) 00000100:00000040:0.0:1786177273.045180:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.045187:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177273.045190:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177273.045193:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177273.045195:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235597312, offset 0 00000400:00000200:0.0:1786177273.045200:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177273.045204:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177273.045207:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177273.045210:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177273.045213:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177273.045216:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177273.045218:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177273.045221:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426733:426733:256:4294967295] 192.168.202.132@tcp LPNI seq info [426695:426695:8:4294967295] 00000400:00000200:0.0:1786177273.045225:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177273.045231:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177273.045236:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177273.045240:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177273.045244:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177273.045246:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1786177273.045250:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177273.045253:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1786177273.045257:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177273.045259:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177273.045262:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177273.045265:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177273.045268:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177273.045270:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177273.045273:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177273.045276:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.045284:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177273.045287:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.045294:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177273.045295:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177273.045299:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177273.045302:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177273.045396:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177273.045400:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306619918 00000100:00000001:0.0:1786177273.045403:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306623745 00000100:00000001:0.0:1786177273.045405:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306619918 00000100:00000001:0.0:1786177273.045407:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306623745 00000100:00000001:0.0:1786177273.045409:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177273.045411:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177273.045413:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177273.045418:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177273/real 1786177273] req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177273.045427:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177273.045429:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177273.045430:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177273.045433:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177273.045435:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177273.045437:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177273.045440:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177273.045448:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.045461:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177273.045463:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.045465:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177273.045466:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177273.045470:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.045476:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177273.045480:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.045487:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177273.045490:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.045491:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177273.045492:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.045494:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177273.045496:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177273.045498:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177273.045502:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177273.045504:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177273.045508:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177273.045516:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597312:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177273.045520:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.045522:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177273.045525:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735f80 x1872930235597312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177273.045531:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177273.045532:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177273.045535:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177273.045537:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177273.045539:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177273.045541:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177273.045543:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177273.045544:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:0.0:1786177273.045546:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177273.045548:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177273.045549:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735f80. 00000100:00000001:0.0:1786177273.045551:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177273.045553:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.045554:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177273.045558:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177273.045559:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177273.045561:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177273.045563:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177273.484757:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:1.0:1786177273.484765:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:1.0:1786177273.484770:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1786177273.485137:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1786177273.485144:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786177273.485159:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786177273.485164:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786177273.485166:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.549104:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177273.549113:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177273.549115:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177273.549120:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177273.549123:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177273.549126:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.549128:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177273.549131:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.549134:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177273.932524:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177273.932532:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177273.932534:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177273.932537:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177273.932542:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177273.932544:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177273.932546:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177273.932548:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177273.932552:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177273.932554:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177273.932557:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177273.932559:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177273.932561:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177273.932562:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177273.932564:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177273.932565:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177273.932567:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177273.932570:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177273.932673:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177273.932678:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177273.932680:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177273.932684:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177273.932688:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177273.932691:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.932693:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177273.932696:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177273.932698:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177273.932765:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177273.932768:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177273.932772:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177274.572713:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177274.572727:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177274.572730:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177274.572737:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177274.572743:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177274.572747:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177274.572750:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177274.572753:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177274.572757:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177274.956458:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177274.956463:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177274.956467:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177274.956468:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177274.956470:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177274.956475:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177274.956475:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177274.956476:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177274.956479:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177274.956480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177274.956481:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177274.956483:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177274.956486:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177274.956488:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177274.956493:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177274.956560:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177274.956565:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177274.956569:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177275.596884:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177275.596891:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177275.596893:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177275.596900:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177275.596903:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177275.596906:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177275.596908:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177275.596911:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177275.596914:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177275.980474:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177275.980481:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177275.980484:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177275.980489:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177275.980493:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177275.980496:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177275.980499:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177275.980501:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177275.980504:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177275.981234:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177275.981248:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177275.981255:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177275.981256:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177275.981260:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177275.981261:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177275.981329:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177275.981333:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177275.981337:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177276.620751:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177276.620761:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177276.620764:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177276.620770:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177276.620774:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177276.620779:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177276.620781:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177276.620784:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177276.620788:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177276.940554:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306623745, at 4306623808 00000100:00000040:1.0:1786177276.941295:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306623745, at 4306623808 00000100:00000010:1.0:1786177276.941313:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c624e0. 00000100:00000010:0.0:1786177276.941594:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62ea0. 00000100:00000001:1.0:1786177277.004520:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177277.004532:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177277.004535:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177277.004542:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177277.004546:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177277.004551:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177277.004554:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177277.004558:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177277.004570:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177277.004785:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177277.004789:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177277.004794:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177277.004795:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177277.004799:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177277.004800:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177277.005061:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177277.005066:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177277.005069:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177277.644506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177277.644518:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177277.644521:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177277.644529:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177277.644534:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177277.644540:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177277.644543:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177277.644548:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177277.644552:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177278.028821:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177278.028826:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177278.028830:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177278.028831:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177278.028833:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177278.028834:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177278.028915:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177278.028917:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177278.028920:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177278.029115:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177278.029118:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177278.029120:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177278.029123:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177278.029125:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177278.029128:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177278.029130:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177278.029132:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177278.029134:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177278.156479:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177278.156489:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177278.156492:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177278.156496:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177278.156501:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177278.156508:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11952 00000400:00000200:2.0:1786177278.156518:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177278.156524:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177278.156527:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177278.156531:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.156534:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177278.156536:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.156538:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177278.156542:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177278.156544:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177278.156547:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177278.156565:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177278.156569:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177278.156572:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177278.156575:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.156577:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177278.156579:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.156581:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177278.156584:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177278.156587:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177278.156589:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177278.156592:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177278.156596:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177278.156598:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.156600:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177278.156603:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177278.156612:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000020:00000040:2.0:1786177278.156616:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177278.156620:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177278.156621:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.156624:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177278.156627:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177278.156630:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177278.156637:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9000. 00000100:00000001:2.0:1786177278.156643:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177278.156653:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177278.156662:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177278.156666:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177278.156671:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.156740:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177278.156744:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177278.156747:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177278.156753:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.156761:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177278.156763:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177278.156766:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177278.156769:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177278.156771:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177278.156775:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235597440:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177278.156779:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177278.156783:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177278.156785:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177278.156799:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177278.156805:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177278.156807:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177278.156812:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177278.156816:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177278.156822:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177278.156830:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235597440, portal 25 00000100:00000001:0.0:1786177278.156833:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177278.156835:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469625472 : -110371239926144 : ffff9b9e31736680) 00000100:00000040:0.0:1786177278.156841:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.156848:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177278.156852:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177278.156854:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177278.156857:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235597440, offset 0 00000400:00000200:0.0:1786177278.156863:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177278.156871:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177278.156876:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177278.156880:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177278.156884:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177278.156887:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177278.156890:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177278.156893:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426734:426734:256:4294967295] 192.168.202.132@tcp LPNI seq info [426696:426696:8:4294967295] 00000400:00000200:0.0:1786177278.156898:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177278.156906:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177278.156912:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177278.156918:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177278.156925:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177278.156928:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177278.156933:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177278.156937:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177278.156941:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177278.156944:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177278.156949:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177278.156953:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177278.156959:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177278.156961:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177278.156965:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177278.156968:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177278.156993:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177278.156997:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177278.157000:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177278.157001:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177278.157003:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177278.157006:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177278.157008:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11952 00000400:00000200:2.0:1786177278.157012:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177278.157015:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177278.157017:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177278.157020:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157022:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177278.157024:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157025:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177278.157027:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177278.157029:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177278.157030:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177278.157032:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177278.157034:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177278.157037:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177278.157039:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157041:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177278.157043:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157045:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177278.157047:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177278.157049:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157050:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177278.157052:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177278.157056:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177278.157058:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157059:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177278.157062:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177278.157064:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157065:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177278.157068:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 00000020:00000040:2.0:1786177278.157069:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177278.157071:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177278.157071:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157073:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177278.157074:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177278.157075:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177278.157078:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aabc00. 00000100:00000001:2.0:1786177278.157091:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177278.157094:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177278.157099:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177278.157101:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177278.157103:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157191:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177278.157195:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157200:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.157201:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177278.157207:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177278.157210:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177278.157212:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306628864 00000100:00000010:0.0:1786177278.157215:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5f00. 00000100:00000001:0.0:1786177278.157220:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177278.157226:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306628864 00000100:00000010:0.0:1786177278.157229:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d58a0. 00000100:00000001:0.0:1786177278.157232:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177278.157235:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177278.157238:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.157240:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177278.157244:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177278/real 1786177278] req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177278.157254:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177278.157256:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177278.157258:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177278.157260:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177278.157262:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177278.157265:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177278.157268:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177278.157274:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157278:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177278.157279:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.157280:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177278.157281:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177278.157284:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157288:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177278.157290:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157295:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177278.157297:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.157298:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177278.157299:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.157300:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177278.157302:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177278.157303:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177278.157305:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177278.157307:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177278.157309:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177278.157313:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235597440:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177278.157317:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.157318:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177278.157320:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736680 x1872930235597440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157324:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177278.157326:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177278.157327:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177278.157329:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177278.157330:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177278.157331:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177278.157333:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177278.157335:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:0.0:1786177278.157337:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177278.157338:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177278.157339:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736680. 00000100:00000001:0.0:1786177278.157341:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177278.157342:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.157344:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.157349:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177278.157350:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.157352:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177278.157353:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177278.157354:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177278.157356:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157386:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177278.157387:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177278.157388:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177278.157389:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177278.157390:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177278.157392:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597568:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177278.157394:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177278.157395:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177278.157396:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177278.157397:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177278.157400:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177278.157401:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177278.157404:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177278.157405:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177278.157408:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177278.157410:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235597568, portal 10 00000100:00000001:0.0:1786177278.157411:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177278.157412:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469626368 : -110371239925248 : ffff9b9e31736a00) 00000100:00000040:0.0:1786177278.157416:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157420:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177278.157422:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177278.157423:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177278.157424:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235597568, offset 0 00000400:00000200:0.0:1786177278.157426:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177278.157429:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177278.157432:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177278.157434:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177278.157435:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177278.157437:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177278.157438:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177278.157440:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426735:426735:256:4294967295] 192.168.202.132@tcp LPNI seq info [426697:426697:8:4294967295] 00000400:00000200:0.0:1786177278.157442:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177278.157446:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177278.157449:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177278.157450:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177278.157453:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177278.157455:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177278.157457:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177278.157459:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177278.157461:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177278.157462:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177278.157464:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177278.157466:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177278.157468:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177278.157470:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177278.157472:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177278.157475:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157482:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177278.157485:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157491:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.157493:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177278.157496:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177278.157499:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177278.157502:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177278.157504:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306625027 00000100:00000001:0.0:1786177278.157508:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306628864 00000100:00000001:0.0:1786177278.157510:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306625027 00000100:00000001:0.0:1786177278.157512:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306628864 00000100:00000001:0.0:1786177278.157514:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177278.157515:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.157517:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177278.157520:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177278/real 1786177278] req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177278.157525:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177278.157525:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177278.157527:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177278.157528:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177278.157529:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177278.157530:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177278.157532:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177278.157537:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157540:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177278.157541:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.157542:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177278.157543:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177278.157544:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157548:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177278.157563:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157570:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177278.157576:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.157578:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177278.157579:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.157581:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177278.157584:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177278.157586:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177278.157590:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177278.157592:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177278.157596:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177278.157603:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597568:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177278.157605:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.157607:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.157609:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1786177278.157609:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1786177278.157612:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736a00 x1872930235597568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177278.157612:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177278.157614:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177278.157615:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177278.157618:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1786177278.157618:0:191280:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1786177278.157619:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:2.0:1786177278.157621:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11952 02000000:00000010:0.0:1786177278.157622:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177278.157624:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177278.157625:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:2.0:1786177278.157625:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:0.0:1786177278.157626:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000200:2.0:1786177278.157628:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000020:00000001:0.0:1786177278.157629:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177278.157631:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aabc00. 00000100:00080000:2.0:1786177278.157631:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1786177278.157633:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177278.157634:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:2.0:1786177278.157634:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000010:0.0:1786177278.157636:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736a00. 00000100:00000001:2.0:1786177278.157636:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1786177278.157638:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000040:2.0:1786177278.157638:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177278.157640:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177278.157640:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177278.157641:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1786177278.157642:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177278.157643:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000100:00000001:0.0:1786177278.157644:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177278.157645:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177278.157645:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:0.0:1786177278.157647:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1786177278.157647:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000100:00000001:0.0:1786177278.157649:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177278.157649:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177278.157651:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177278.157653:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157655:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177278.157657:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157659:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177278.157661:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177278.157663:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157665:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177278.157667:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177278.157669:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177278.157671:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157673:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177278.157675:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177278.157678:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157679:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177278.157683:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 00000020:00000040:2.0:1786177278.157684:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177278.157687:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177278.157688:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157689:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177278.157691:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177278.157692:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177278.157697:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aabc00. 00000100:00000001:2.0:1786177278.157701:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177278.157705:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177278.157712:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177278.157715:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177278.157718:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177278.157727:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177278.157729:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177278.157733:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177278.157734:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177278.157736:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177278.157738:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177278.157741:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11952 00000400:00000200:2.0:1786177278.157744:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177278.157747:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177278.157749:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177278.157752:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157754:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177278.157755:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157757:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177278.157759:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177278.157760:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177278.157762:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177278.157764:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177278.157766:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177278.157768:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177278.157770:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157772:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177278.157774:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157776:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177278.157778:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177278.157780:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157781:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177278.157783:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177278.157786:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177278.157787:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157789:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177278.157791:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177278.157793:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157795:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177278.157798:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000020:00000040:2.0:1786177278.157800:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177278.157801:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177278.157802:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157804:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177278.157805:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177278.157807:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177278.157810:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9000. 00000100:00000001:2.0:1786177278.157812:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177278.157816:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177278.157821:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177278.157823:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177278.157825:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177278.157832:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177278.157834:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177278.157836:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177278.157837:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177278.157844:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177278.157847:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177278.157849:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11952 00000400:00000200:2.0:1786177278.157853:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177278.157855:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177278.157857:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177278.157860:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157862:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177278.157863:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157865:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177278.157868:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177278.157869:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177278.157870:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177278.157872:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177278.157874:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177278.157876:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177278.157879:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157881:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177278.157883:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157885:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177278.157887:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177278.157889:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177278.157892:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177278.157893:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177278.157896:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177278.157898:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157900:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177278.157902:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177278.157904:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157905:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177278.157909:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000020:00000040:2.0:1786177278.157911:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177278.157914:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177278.157915:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.157917:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177278.157917:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177278.157918:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1786177278.157919:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1786177278.157920:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.157921:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000010:2.0:1786177278.157923:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000040:0.0:1786177278.157925:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177278.157926:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177278.157930:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177278.157932:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177278.157933:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177278.157935:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177278.157937:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177278.157937:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1786177278.157939:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157939:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177278.157942:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597696:192.168.202.132@tcp:38:kworker.0 00000100:00000040:2.0:1786177278.157943:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157946:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177278.157948:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177278.157949:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.157950:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177278.157951:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177278.157953:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 00000100:00000040:2.0:1786177278.157954:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11962) 02000000:00000001:0.0:1786177278.157955:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177278.157958:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177278.157960:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177278.157964:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177278.157967:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235597696, portal 10 00000100:00000001:0.0:1786177278.157968:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177278.157970:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469615616 : -110371239936000 : ffff9b9e31734000) 00000100:00000040:0.0:1786177278.157975:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.157981:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177278.157983:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177278.157985:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177278.157987:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235597696, offset 0 00000400:00000200:0.0:1786177278.157991:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177278.157995:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177278.157998:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177278.158001:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177278.158004:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177278.158006:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177278.158008:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177278.158011:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426736:426736:256:4294967295] 192.168.202.132@tcp LPNI seq info [426698:426698:8:4294967295] 00000400:00000200:0.0:1786177278.158015:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177278.158021:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177278.158026:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177278.158029:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177278.158032:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177278.158035:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177278.158039:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177278.158041:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177278.158045:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177278.158046:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177278.158049:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177278.158052:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177278.158055:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177278.158057:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177278.158059:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177278.158063:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158071:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177278.158073:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158179:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.158181:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177278.158184:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177278.158186:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177278.158189:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177278.158191:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306625028 00000100:00000001:0.0:1786177278.158194:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306628864 00000100:00000001:0.0:1786177278.158196:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306625028 00000100:00000001:0.0:1786177278.158198:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306628864 00000100:00000001:0.0:1786177278.158200:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177278.158202:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.158203:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177278.158208:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177278/real 1786177278] req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177278.158216:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177278.158217:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177278.158218:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177278.158220:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177278.158222:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177278.158224:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177278.158227:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177278.158236:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158242:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177278.158244:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158246:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177278.158248:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177278.158251:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158258:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177278.158262:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158268:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177278.158271:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158272:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177278.158273:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158276:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177278.158278:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177278.158280:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177278.158283:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177278.158285:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177278.158289:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177278.158297:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597696:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177278.158300:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177278.158303:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158310:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177278.158311:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177278.158313:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177278.158314:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177278.158316:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177278.158319:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597824:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177278.158322:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177278.158324:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177278.158325:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177278.158327:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177278.158330:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177278.158332:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177278.158335:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177278.158337:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177278.158339:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177278.158341:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235597824, portal 4 00000100:00000001:0.0:1786177278.158343:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177278.158343:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469624576 : -110371239927040 : ffff9b9e31736300) 00000100:00000040:0.0:1786177278.158347:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158351:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177278.158353:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177278.158354:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177278.158355:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235597824, offset 0 00000400:00000200:0.0:1786177278.158372:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177278.158376:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177278.158379:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177278.158382:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177278.158384:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177278.158386:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177278.158388:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177278.158391:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426737:426737:256:4294967295] 192.168.202.132@tcp LPNI seq info [426699:426699:8:4294967295] 00000400:00000200:0.0:1786177278.158395:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177278.158401:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177278.158405:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177278.158408:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177278.158412:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177278.158415:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177278.158418:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177278.158421:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177278.158424:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177278.158426:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177278.158429:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177278.158432:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177278.158434:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177278.158436:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177278.158439:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177278.158442:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158449:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177278.158451:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158463:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.158464:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177278.158468:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177278.158470:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177278.158473:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177278.158475:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306625029 00000100:00000001:0.0:1786177278.158477:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306628864 00000100:00000001:0.0:1786177278.158479:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306625029 00000100:00000001:0.0:1786177278.158481:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306628864 00000100:00000001:0.0:1786177278.158483:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177278.158485:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.158487:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177278.158491:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177278/real 1786177278] req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177278.158498:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177278.158499:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177278.158501:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177278.158503:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177278.158505:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177278.158506:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177278.158510:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177278.158517:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158524:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177278.158525:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158526:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177278.158528:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177278.158531:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158539:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177278.158542:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158548:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177278.158560:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158561:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177278.158562:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158564:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177278.158566:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177278.158568:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177278.158571:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177278.158573:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177278.158578:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177278.158585:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597824:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177278.158589:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158591:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177278.158594:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734000 x1872930235597696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158600:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177278.158602:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177278.158604:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177278.158606:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177278.158608:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177278.158609:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177278.158611:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177278.158612:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:0.0:1786177278.158615:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177278.158617:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177278.158618:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734000. 00000100:00000001:0.0:1786177278.158620:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177278.158621:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158623:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177278.158626:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736300 x1872930235597824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158632:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177278.158634:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177278.158635:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177278.158637:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177278.158639:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177278.158640:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177278.158642:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177278.158643:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:0.0:1786177278.158645:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177278.158646:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177278.158648:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736300. 00000100:00000001:0.0:1786177278.158650:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177278.158651:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158653:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158656:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177278.158658:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.158660:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177278.158661:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177278.158662:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177278.158666:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158672:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177278.158673:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177278.158675:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177278.158676:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177278.158678:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177278.158681:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597952:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177278.158687:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177278.158688:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177278.158690:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177278.158691:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177278.158694:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177278.158696:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177278.158699:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177278.158701:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177278.158705:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177278.158708:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235597952, portal 4 00000100:00000001:0.0:1786177278.158709:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177278.158710:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469616512 : -110371239935104 : ffff9b9e31734380) 00000100:00000040:0.0:1786177278.158716:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158722:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177278.158724:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177278.158726:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177278.158728:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235597952, offset 0 00000400:00000200:0.0:1786177278.158732:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177278.158736:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177278.158739:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177278.158742:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177278.158744:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177278.158747:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177278.158749:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177278.158751:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426738:426738:256:4294967295] 192.168.202.132@tcp LPNI seq info [426700:426700:8:4294967295] 00000400:00000200:0.0:1786177278.158755:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177278.158761:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177278.158765:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177278.158768:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177278.158772:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177278.158775:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177278.158779:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177278.158781:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177278.158785:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177278.158787:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177278.158852:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177278.158857:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177278.158860:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177278.158862:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177278.158864:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177278.158868:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158875:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177278.158878:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177303 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158884:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.158885:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177278.158889:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177278.158892:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177278.158895:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177278.158897:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306625031 00000100:00000001:0.0:1786177278.158899:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306628864 00000100:00000001:0.0:1786177278.158901:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306625031 00000100:00000001:0.0:1786177278.158903:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306628864 00000100:00000001:0.0:1786177278.158905:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177278.158907:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.158909:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177278.158912:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177278/real 1786177278] req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177278.158919:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177278.158921:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177278.158923:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177278.158925:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177278.158927:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177278.158929:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177278.158932:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177278.158939:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158946:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177278.158947:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158949:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177278.158950:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177278.158953:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158960:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177278.158964:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.158970:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177278.158972:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158974:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177278.158975:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.158976:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177278.158978:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177278.158980:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177278.158983:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177278.158986:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177278.158989:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177278.158996:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235597952:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177278.159000:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.159002:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177278.159005:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734380 x1872930235597952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177303 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177278.159011:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177278.159012:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177278.159014:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177278.159017:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177278.159018:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177278.159020:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177278.159021:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177278.159023:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:0.0:1786177278.159025:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177278.159026:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177278.159028:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734380. 00000100:00000001:0.0:1786177278.159030:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177278.159032:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.159033:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177278.159037:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177278.159038:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177278.159040:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177278.159041:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177278.668543:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177278.668560:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177278.668563:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177278.668568:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177278.668576:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177278.668580:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177278.668582:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177278.668585:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177278.668588:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177279.052524:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177279.052531:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177279.052537:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177279.052538:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177279.052569:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177279.052571:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177279.053384:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177279.053388:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177279.053391:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177279.054797:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177279.054803:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177279.054805:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177279.054810:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177279.054813:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177279.054817:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177279.054819:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177279.054822:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177279.054824:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177279.692867:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177279.692876:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177279.692879:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177279.692886:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177279.692890:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177279.692895:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177279.692899:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177279.692902:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177279.692930:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177280.076436:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177280.076441:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177280.076446:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177280.076448:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177280.076453:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177280.076454:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177280.076457:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177280.076459:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177280.076461:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177280.076464:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177280.076467:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177280.076472:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177280.076476:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177280.076478:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177280.076482:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177280.076571:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177280.076576:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177280.076579:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177280.716472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177280.716483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177280.716486:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177280.716492:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177280.716498:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177280.716502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177280.716506:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177280.716509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177280.716513:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177281.100506:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177281.100510:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177281.100519:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177281.100520:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177281.100533:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177281.100540:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177281.100544:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177281.100549:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177281.100552:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177281.100555:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177281.100558:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177281.100730:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177281.100734:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177281.100736:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177281.100739:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177281.100742:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177281.100744:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177281.100746:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177281.100748:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177281.100752:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177281.100753:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177281.100804:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177281.100808:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177281.100871:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177281.740444:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177281.740452:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177281.740454:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177281.740460:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177281.740463:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177281.740467:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177281.740470:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177281.740472:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177281.740476:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177282.060430:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306628864, at 4306628928 00000100:00000040:1.0:1786177282.060430:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306628864, at 4306628928 00000100:00000010:1.0:1786177282.060445:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d58a0. 00000100:00000010:0.0:1786177282.060446:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5f00. 00000100:00000001:1.0:1786177282.124464:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177282.124468:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177282.124471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177282.124471:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177282.124473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177282.124476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177282.124477:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177282.124478:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177282.124480:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177282.124481:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177282.124482:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177282.124483:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177282.124486:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177282.124489:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177282.124491:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177282.125440:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177282.125444:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177282.125449:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177282.764632:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177282.764641:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177282.764643:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177282.764648:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177282.764652:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177282.764656:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177282.764658:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177282.764660:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177282.764664:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177283.148487:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177283.148495:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177283.148497:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177283.148513:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177283.148517:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177283.148521:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177283.148524:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177283.148526:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177283.148529:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177283.148579:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177283.148583:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177283.148587:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177283.148588:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177283.148592:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177283.148593:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177283.148700:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177283.148705:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177283.148708:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177283.276489:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177283.276498:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177283.276515:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177283.276518:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177283.276522:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177283.276529:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11957 00000400:00000200:2.0:1786177283.276536:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177283.276542:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177283.276545:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177283.276548:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276551:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177283.276553:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276554:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177283.276557:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177283.276559:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177283.276561:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177283.276564:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177283.276566:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177283.276568:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177283.276570:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276571:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177283.276573:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276574:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177283.276576:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177283.276578:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276579:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177283.276582:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177283.276584:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177283.276586:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276588:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177283.276590:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177283.276597:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 00000020:00000040:2.0:1786177283.276600:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177283.276603:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177283.276604:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276606:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177283.276609:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177283.276611:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177283.276616:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786177283.276621:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177283.276628:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177283.276638:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177283.276642:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177283.276647:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177283.276671:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177283.276675:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177283.276678:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177283.276679:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177283.276681:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177283.276683:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177283.276685:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11957 00000400:00000200:2.0:1786177283.276688:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177283.276689:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177283.276691:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177283.276693:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276694:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177283.276696:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276697:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177283.276699:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177283.276700:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177283.276701:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177283.276702:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177283.276703:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177283.276704:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177283.276706:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276707:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177283.276708:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276709:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177283.276710:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177283.276711:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276713:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177283.276714:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177283.276718:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177283.276719:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276720:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177283.276724:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177283.276726:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276727:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177283.276729:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786177283.276731:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177283.276732:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177283.276733:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276734:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177283.276735:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177283.276736:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177283.276739:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177283.276741:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177283.276744:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177283.276750:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177283.276751:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177283.276754:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177283.276758:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177283.276760:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177283.276762:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177283.276763:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177283.276764:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177283.276765:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177283.276767:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11957 00000400:00000200:2.0:1786177283.276776:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177283.276779:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177283.276781:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177283.276783:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276785:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177283.276787:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276788:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177283.276790:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177283.276791:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177283.276793:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177283.276794:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177283.276796:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177283.276798:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177283.276801:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276802:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177283.276804:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276806:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177283.276809:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177283.276811:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276814:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177283.276815:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177283.276818:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177283.276820:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276821:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177283.276823:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177283.276824:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276826:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177283.276828:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786177283.276830:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177283.276831:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177283.276832:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276833:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177283.276834:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177283.276835:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177283.276838:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:2.0:1786177283.276840:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177283.276843:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177283.276847:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177283.276849:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177283.276851:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177283.276855:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177283.276857:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177283.276859:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177283.276860:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177283.276861:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177283.276863:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177283.276865:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11957 00000400:00000200:2.0:1786177283.276867:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177283.276869:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177283.276871:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177283.276873:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276874:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177283.276875:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276876:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177283.276878:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177283.276878:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177283.276879:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177283.276880:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177283.276881:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177283.276882:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177283.276884:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276885:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177283.276886:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276887:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177283.276888:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177283.276889:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276890:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177283.276891:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177283.276893:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177283.276894:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276895:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177283.276896:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177283.276897:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276898:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177283.276900:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000020:00000040:2.0:1786177283.276901:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177283.276902:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177283.276903:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276904:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177283.276905:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177283.276906:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177283.276908:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:2.0:1786177283.276909:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177283.276912:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177283.276915:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177283.276917:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177283.276919:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177283.276923:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177283.276924:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177283.276926:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177283.276927:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177283.276927:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177283.276929:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177283.276930:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11957 00000400:00000200:2.0:1786177283.276931:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177283.276933:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177283.276934:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177283.276935:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276936:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177283.276937:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276938:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177283.276939:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177283.276940:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177283.276940:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177283.276941:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177283.276942:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177283.276944:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177283.276946:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276947:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177283.276948:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276949:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177283.276950:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177283.276951:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177283.276952:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177283.276953:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177283.276954:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177283.276955:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276956:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177283.276957:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177283.276959:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276959:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177283.276961:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737480. 00000020:00000040:2.0:1786177283.276962:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177283.276963:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177283.276964:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.276965:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177283.276965:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177283.276966:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177283.276968:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000001:2.0:1786177283.276969:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177283.276971:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177283.276975:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177283.276975:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177283.276977:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046bba14a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177283.276983:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177283.276988:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11967) 00000100:00000001:0.0:1786177283.277041:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177283.277044:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177283.277046:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177283.277051:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277056:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177283.277057:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177283.277060:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177283.277062:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177283.277063:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177283.277065:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235598080:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177283.277070:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177283.277073:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177283.277074:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177283.277076:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177283.277080:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177283.277082:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177283.277087:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177283.277090:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177283.277097:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177283.277103:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235598080, portal 25 00000100:00000001:0.0:1786177283.277106:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177283.277107:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469630848 : -110371239920768 : ffff9b9e31737b80) 00000100:00000040:0.0:1786177283.277113:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277120:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177283.277122:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177283.277125:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177283.277128:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235598080, offset 0 00000400:00000200:0.0:1786177283.277132:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177283.277138:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177283.277143:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177283.277147:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177283.277149:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177283.277152:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177283.277154:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177283.277157:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426739:426739:256:4294967295] 192.168.202.132@tcp LPNI seq info [426701:426701:8:4294967295] 00000400:00000200:0.0:1786177283.277160:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177283.277165:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177283.277170:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177283.277175:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177283.277181:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177283.277185:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177283.277190:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177283.277193:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177283.277197:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177283.277200:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177283.277206:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177283.277210:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177283.277216:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177283.277218:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177283.277221:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177283.277223:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277230:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177283.277233:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277237:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177283.277239:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177283.277243:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177283.277246:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177283.277247:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306633984 00000100:00000010:0.0:1786177283.277251:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5f00. 00000100:00000001:0.0:1786177283.277255:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177283.277259:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306633984 00000100:00000010:0.0:1786177283.277261:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d52a0. 00000100:00000001:0.0:1786177283.277263:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177283.277265:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177283.277267:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177283.277268:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177283.277271:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177283/real 1786177283] req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177283.277277:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177283.277278:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177283.277280:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177283.277281:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177283.277282:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177283.277283:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177283.277285:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177283.277290:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277294:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177283.277296:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277297:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177283.277298:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177283.277300:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277304:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177283.277307:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277310:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177283.277312:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277313:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177283.277314:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277315:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177283.277316:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177283.277318:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177283.277321:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177283.277322:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177283.277325:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177283.277329:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235598080:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177283.277333:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177283.277336:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277339:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177283.277340:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177283.277341:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177283.277342:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177283.277343:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177283.277345:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598208:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177283.277347:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177283.277348:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177283.277349:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177283.277350:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177283.277352:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177283.277353:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177283.277355:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177283.277373:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177283.277376:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177283.277378:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235598208, portal 10 00000100:00000001:0.0:1786177283.277379:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177283.277380:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620992 : -110371239930624 : ffff9b9e31735500) 00000100:00000040:0.0:1786177283.277384:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277388:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177283.277389:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177283.277390:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177283.277392:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235598208, offset 0 00000400:00000200:0.0:1786177283.277394:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177283.277396:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177283.277398:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177283.277400:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177283.277402:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177283.277404:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177283.277405:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177283.277407:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426740:426740:256:4294967295] 192.168.202.132@tcp LPNI seq info [426702:426702:8:4294967295] 00000400:00000200:0.0:1786177283.277409:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177283.277413:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177283.277415:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177283.277417:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177283.277419:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177283.277421:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177283.277424:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177283.277426:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177283.277428:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177283.277429:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177283.277431:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177283.277433:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177283.277435:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177283.277436:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177283.277437:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177283.277439:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277444:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177283.277446:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277449:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177283.277450:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177283.277452:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177283.277454:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177283.277455:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177283.277456:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306630148 00000100:00000001:0.0:1786177283.277458:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306633984 00000100:00000001:0.0:1786177283.277460:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306630148 00000100:00000001:0.0:1786177283.277461:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306633984 00000100:00000001:0.0:1786177283.277463:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177283.277464:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177283.277465:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177283.277467:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177283/real 1786177283] req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177283.277471:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177283.277472:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177283.277473:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177283.277474:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177283.277475:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177283.277476:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177283.277478:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177283.277482:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277485:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177283.277486:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277487:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177283.277488:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177283.277489:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277493:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177283.277495:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277499:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177283.277512:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277514:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177283.277515:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277517:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177283.277519:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177283.277521:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177283.277524:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177283.277525:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177283.277528:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177283.277532:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598208:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177283.277534:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177283.277536:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277540:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177283.277541:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177283.277542:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177283.277543:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177283.277544:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177283.277546:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598336:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177283.277548:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177283.277549:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177283.277550:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177283.277551:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177283.277554:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177283.277556:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177283.277558:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177283.277559:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177283.277561:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177283.277563:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235598336, portal 10 00000100:00000001:0.0:1786177283.277564:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177283.277565:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469622784 : -110371239928832 : ffff9b9e31735c00) 00000100:00000040:0.0:1786177283.277569:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277573:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177283.277574:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177283.277575:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177283.277577:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235598336, offset 0 00000400:00000200:0.0:1786177283.277579:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177283.277581:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177283.277583:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177283.277585:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177283.277586:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177283.277588:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177283.277589:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177283.277591:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426741:426741:256:4294967295] 192.168.202.132@tcp LPNI seq info [426703:426703:8:4294967295] 00000400:00000200:0.0:1786177283.277593:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177283.277596:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177283.277599:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177283.277600:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177283.277603:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177283.277604:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177283.277607:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177283.277608:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177283.277612:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177283.277613:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177283.277616:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177283.277619:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177283.277622:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177283.277623:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177283.277625:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177283.277628:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277638:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177283.277640:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277651:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177283.277652:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177283.277655:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177283.277657:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177283.277660:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177283.277661:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306630148 00000100:00000001:0.0:1786177283.277664:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306633984 00000100:00000001:0.0:1786177283.277666:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306630148 00000100:00000001:0.0:1786177283.277668:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306633984 00000100:00000001:0.0:1786177283.277669:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177283.277671:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177283.277673:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177283.277676:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177283/real 1786177283] req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177283.277682:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177283.277683:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177283.277684:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177283.277686:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177283.277688:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177283.277689:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177283.277692:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177283.277699:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277704:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177283.277705:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277707:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177283.277708:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177283.277711:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277717:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177283.277721:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277727:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177283.277729:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277730:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177283.277731:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277733:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177283.277734:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177283.277737:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177283.277740:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177283.277741:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177283.277745:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177283.277753:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598336:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177283.277756:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177283.277759:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277765:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177283.277766:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177283.277768:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177283.277770:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177283.277771:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177283.277774:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598464:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177283.277777:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177283.277779:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177283.277781:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177283.277782:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177283.277785:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177283.277787:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177283.277791:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177283.277793:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177283.277797:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177283.277800:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235598464, portal 4 00000100:00000001:0.0:1786177283.277801:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177283.277802:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469621888 : -110371239929728 : ffff9b9e31735880) 00000100:00000040:0.0:1786177283.277807:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277813:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177283.277815:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177283.277817:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177283.277819:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235598464, offset 0 00000400:00000200:0.0:1786177283.277823:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177283.277826:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177283.277830:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177283.277832:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177283.277835:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177283.277837:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177283.277839:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177283.277842:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426742:426742:256:4294967295] 192.168.202.132@tcp LPNI seq info [426704:426704:8:4294967295] 00000400:00000200:0.0:1786177283.277846:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177283.277851:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177283.277854:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177283.277856:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177283.277858:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177283.277860:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177283.277863:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177283.277864:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177283.277867:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177283.277869:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177283.277871:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177283.277873:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177283.277875:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177283.277876:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177283.277878:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177283.277880:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277885:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177283.277887:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277891:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177283.277892:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177283.277894:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177283.277896:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177283.277899:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177283.277901:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306630148 00000100:00000001:0.0:1786177283.277904:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306633984 00000100:00000001:0.0:1786177283.277906:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306630148 00000100:00000001:0.0:1786177283.277908:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306633984 00000100:00000001:0.0:1786177283.277910:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177283.277911:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177283.277913:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177283.277916:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177283/real 1786177283] req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177283.277924:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177283.277925:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177283.277927:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177283.277929:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177283.277931:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177283.277933:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177283.277936:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177283.277944:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277948:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177283.277949:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277950:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177283.277952:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177283.277954:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277958:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177283.277960:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277963:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177283.277964:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277965:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177283.277966:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.277967:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177283.277968:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177283.277969:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177283.277971:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177283.277972:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177283.277974:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177283.277978:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598464:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177283.277980:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177283.277983:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.277989:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177283.277990:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177283.277992:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177283.277994:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177283.277996:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177283.277998:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598592:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177283.278001:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177283.278003:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177283.278004:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177283.278006:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177283.278009:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177283.278010:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177283.278014:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177283.278016:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177283.278019:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177283.278021:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235598592, portal 4 00000100:00000001:0.0:1786177283.278023:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177283.278024:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469629056 : -110371239922560 : ffff9b9e31737480) 00000100:00000040:0.0:1786177283.278030:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.278034:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177283.278035:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177283.278036:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177283.278038:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235598592, offset 0 00000400:00000200:0.0:1786177283.278040:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177283.278043:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177283.278045:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177283.278047:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177283.278048:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177283.278050:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177283.278051:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177283.278053:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426743:426743:256:4294967295] 192.168.202.132@tcp LPNI seq info [426705:426705:8:4294967295] 00000400:00000200:0.0:1786177283.278055:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177283.278059:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177283.278062:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177283.278064:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177283.278066:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177283.278068:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177283.278070:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177283.278072:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177283.278074:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177283.278075:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177283.278077:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177283.278079:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177283.278081:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177283.278083:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177283.278085:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177283.278088:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.278095:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177283.278098:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177308 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.278104:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177283.278106:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177283.278109:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177283.278111:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177283.278114:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177283.278116:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306630150 00000100:00000001:0.0:1786177283.278119:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306633984 00000100:00000001:0.0:1786177283.278121:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306630150 00000100:00000001:0.0:1786177283.278123:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306633984 00000100:00000001:0.0:1786177283.278125:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177283.278127:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177283.278128:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177283.278131:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177283/real 1786177283] req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177283.278136:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177283.278136:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177283.278138:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177283.278139:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177283.278140:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177283.278141:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177283.278143:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177283.278148:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.278151:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177283.278152:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.278153:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177283.278154:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177283.278155:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.278159:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177283.278161:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.278165:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177283.278167:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.278168:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177283.278168:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.278170:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177283.278171:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177283.278173:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177283.278176:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177283.278178:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177283.278182:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177283.278188:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598592:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177283.278192:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.278194:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177283.278197:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737b80 x1872930235598080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.278203:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177283.278205:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177283.278208:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177283.278210:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177283.278212:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177283.278214:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177283.278216:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177283.278219:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:0.0:1786177283.278223:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177283.278225:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177283.278226:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737b80. 00000100:00000001:0.0:1786177283.278229:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177283.278232:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.278233:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177283.278236:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735500 x1872930235598208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.278242:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177283.278243:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177283.278244:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177283.278246:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177283.278247:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177283.278248:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177283.278249:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177283.278251:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:0.0:1786177283.278253:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177283.278254:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177283.278255:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735500. 00000100:00000001:0.0:1786177283.278257:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177283.278258:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.278260:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177283.278263:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735c00 x1872930235598336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.278268:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177283.278269:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177283.278270:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177283.278272:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177283.278273:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177283.278274:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177283.278276:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177283.278277:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:0.0:1786177283.278279:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177283.278280:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177283.278281:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735c00. 00000100:00000001:0.0:1786177283.278283:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177283.278284:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.278285:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177283.278288:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735880 x1872930235598464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.278292:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177283.278293:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177283.278294:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177283.278296:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177283.278297:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177283.278298:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177283.278300:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177283.278301:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:0.0:1786177283.278303:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177283.278304:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177283.278305:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735880. 00000100:00000001:0.0:1786177283.278307:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177283.278308:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.278309:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177283.278312:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737480 x1872930235598592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177308 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177283.278317:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177283.278318:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177283.278319:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177283.278321:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177283.278322:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177283.278323:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177283.278325:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177283.278326:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:0.0:1786177283.278327:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177283.278328:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177283.278329:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737480. 00000100:00000001:0.0:1786177283.278331:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177283.278332:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.278334:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177283.278341:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177283.278342:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177283.278344:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177283.278346:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177283.788477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177283.788485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177283.788488:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177283.788494:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177283.788518:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177283.788522:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177283.788526:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177283.788528:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177283.788532:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177284.172653:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177284.172663:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177284.172665:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177284.172669:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177284.172671:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177284.172680:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177284.172682:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177284.172685:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177284.172687:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177284.172690:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177284.172695:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177284.172696:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177284.172702:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177284.172705:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177284.172708:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177284.172710:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177284.172714:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177284.172719:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177284.172722:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177284.172727:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177284.172730:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177284.172732:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177284.172734:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177284.172738:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177284.172740:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177284.172742:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177284.172744:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177284.172875:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177284.172880:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177284.172884:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177284.812533:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177284.812543:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177284.812546:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177284.812552:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177284.812557:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177284.812561:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177284.812563:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177284.812565:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177284.812568:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177285.196468:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177285.196476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177285.196478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177285.196510:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177285.196515:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177285.196518:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177285.196528:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177285.196530:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177285.196533:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177285.196611:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177285.196615:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177285.196619:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177285.196621:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177285.196624:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177285.196625:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177285.196677:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177285.196681:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177285.196684:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177285.837423:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177285.837430:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177285.837433:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177285.837438:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177285.837442:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177285.837445:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177285.837448:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177285.837451:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177285.837453:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177286.220595:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177286.220603:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177286.220606:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177286.220611:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177286.220615:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177286.220619:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177286.220622:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177286.220625:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177286.220628:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177286.221560:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177286.221564:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177286.221568:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177286.221569:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177286.221572:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177286.221573:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177286.221634:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177286.221638:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177286.221642:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177286.860656:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177286.860666:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177286.860668:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177286.860674:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177286.860679:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177286.860683:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177286.860686:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177286.860689:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177286.860692:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177287.180498:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306633984, at 4306634048 00000100:00000010:0.0:1786177287.180513:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5f00. 00000100:00000040:1.0:1786177287.180641:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306633984, at 4306634048 00000100:00000010:1.0:1786177287.180648:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d52a0. 00010000:00000001:0.0:1786177287.244435:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177287.244442:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177287.244448:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177287.244449:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177287.244452:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177287.244453:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177287.244569:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177287.244572:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177287.244575:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177287.261689:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177287.261697:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177287.261699:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177287.261704:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177287.261708:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177287.261711:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177287.261713:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177287.261716:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177287.261719:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177287.884741:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177287.884749:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177287.884751:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177287.884756:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177287.884760:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177287.884769:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177287.884772:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177287.884775:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177287.884778:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177288.268552:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177288.268560:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177288.268562:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177288.268568:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177288.268572:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177288.268575:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177288.268578:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177288.268580:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177288.268583:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177288.268801:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177288.268809:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177288.268815:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177288.268817:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177288.268820:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177288.268821:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177288.268887:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177288.268892:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177288.268895:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177288.396615:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177288.396623:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177288.396625:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177288.396628:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177288.396632:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177288.396646:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11962 00000400:00000200:3.0:1786177288.396653:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177288.396658:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177288.396661:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177288.396663:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396665:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177288.396667:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396668:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177288.396671:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177288.396673:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177288.396674:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177288.396677:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177288.396678:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177288.396680:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177288.396682:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396683:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177288.396684:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396692:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177288.396695:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177288.396698:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396699:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177288.396701:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177288.396710:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177288.396713:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.396715:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177288.396717:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177288.396727:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786177288.396730:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177288.396734:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177288.396746:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.396750:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177288.396752:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177288.396756:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177288.396764:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177288.396769:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177288.396777:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177288.396793:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177288.396796:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177288.396799:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177288.396819:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177288.396821:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177288.396823:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177288.396824:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177288.396825:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177288.396827:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177288.396828:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11962 00000400:00000200:3.0:1786177288.396831:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177288.396833:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177288.396835:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177288.396844:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396845:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177288.396847:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396848:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177288.396849:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177288.396850:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177288.396851:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177288.396853:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177288.396854:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177288.396855:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177288.396857:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396858:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177288.396859:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396860:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177288.396862:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177288.396863:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396865:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177288.396866:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177288.396869:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177288.396875:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.396877:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177288.396881:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177288.396884:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.396886:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177288.396899:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786177288.396902:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177288.396904:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177288.396905:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.396907:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177288.396909:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177288.396911:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177288.396915:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786177288.396918:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177288.396924:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177288.396929:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177288.396930:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177288.396933:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177288.396947:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177288.396949:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177288.396951:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177288.396952:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177288.396953:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177288.396954:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177288.396956:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11962 00000400:00000200:3.0:1786177288.396959:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177288.396977:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177288.396979:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177288.396982:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396984:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177288.396986:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.396988:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177288.397001:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177288.397002:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177288.397004:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177288.397006:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177288.397008:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177288.397010:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177288.397013:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.397015:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177288.397017:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.397018:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177288.397021:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177288.397023:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177288.397037:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177288.397039:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177288.397042:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177288.397044:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.397046:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177288.397048:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177288.397050:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.397052:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177288.397055:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786177288.397058:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177288.397060:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177288.397062:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.397063:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177288.397065:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177288.397066:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177288.397070:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786177288.397073:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.397190:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177288.397195:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177288.397197:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177288.397204:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.397212:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177288.397214:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177288.397217:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177288.397219:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177288.397222:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177288.397225:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235598720:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177288.397240:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177288.397244:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177288.397246:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177288.397249:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177288.397254:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177288.397257:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177288.397262:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177288.397266:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177288.397272:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177288.397290:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235598720, portal 25 00000100:00000001:0.0:1786177288.397293:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177288.397295:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280820736 : -110371428730880 : ffff9b9e26327800) 00000100:00000040:0.0:1786177288.397300:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.397308:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177288.397310:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177288.397313:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177288.397316:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235598720, offset 0 00000400:00000200:0.0:1786177288.397322:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177288.397345:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177288.397352:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177288.397376:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177288.397391:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177288.397395:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177288.397398:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177288.397401:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426744:426744:256:4294967295] 192.168.202.132@tcp LPNI seq info [426706:426706:8:4294967295] 00000400:00000200:0.0:1786177288.397406:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177288.397414:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177288.397420:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177288.397424:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177288.397440:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177288.397443:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1786177288.397449:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177288.397474:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1786177288.397489:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177288.397493:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177288.397498:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177288.397503:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177288.397509:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177288.397511:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177288.397515:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177288.397519:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.397528:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177288.397543:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.397552:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.397553:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177288.397560:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177288.397563:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177288.397566:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306639105 00000100:00000010:0.0:1786177288.397571:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5840. 00000100:00000001:0.0:1786177288.397577:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177288.397594:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306639105 00000100:00000010:0.0:1786177288.397597:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5f00. 00000100:00000001:0.0:1786177288.397600:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177288.397603:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177288.397605:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.397608:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177288.397612:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177288/real 1786177288] req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177288.397621:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177288.397623:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177288.397625:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177288.397628:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177288.397629:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177288.397631:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177288.397646:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177288.397655:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.397662:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177288.397663:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.397666:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177288.397668:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177288.397672:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.397678:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177288.397682:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.397700:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177288.397703:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.397705:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177288.397706:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.397708:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177288.397710:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177288.397713:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177288.397717:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177288.397719:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177288.397724:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177288.397732:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235598720:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177288.397747:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177288.397750:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.397757:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177288.397758:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177288.397761:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177288.397763:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177288.397764:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177288.397767:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598848:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177288.397770:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177288.397772:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177288.397774:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177288.397776:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177288.397779:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177288.397781:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177288.397798:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177288.397802:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177288.397805:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177288.397808:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235598848, portal 10 00000100:00000001:0.0:1786177288.397810:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177288.397811:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818048 : -110371428733568 : ffff9b9e26326d80) 00000100:00000040:0.0:1786177288.397816:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.397823:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177288.397825:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177288.397827:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177288.397829:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235598848, offset 0 00000400:00000200:0.0:1786177288.397833:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177288.397850:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177288.397854:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177288.397857:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177288.397860:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177288.397862:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177288.397864:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177288.397867:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426745:426745:256:4294967295] 192.168.202.132@tcp LPNI seq info [426707:426707:8:4294967295] 00000400:00000200:0.0:1786177288.397871:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177288.397889:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177288.397894:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177288.397898:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177288.397901:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177288.397904:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1786177288.397908:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177288.397910:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1786177288.397914:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177288.397916:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177288.397918:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177288.397922:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177288.397938:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177288.397940:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177288.397943:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177288.397947:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.397954:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177288.397957:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786177288.398053:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177288.398061:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177288.398063:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177288.398067:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177288.398075:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177288.398090:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177288.398094:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177288.398095:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177288.398097:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177288.398100:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177288.398103:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11962 00000400:00000200:3.0:1786177288.398107:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177288.398109:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177288.398112:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177288.398114:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.398117:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177288.398118:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.398121:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177288.398123:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177288.398124:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177288.398125:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177288.398139:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177288.398141:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177288.398143:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177288.398146:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.398148:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177288.398149:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.398151:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177288.398153:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177288.398156:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177288.398158:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177288.398160:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177288.398163:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177288.398165:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.398167:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177288.398169:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177288.398171:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.398172:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177288.398186:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786177288.398190:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177288.398192:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177288.398193:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.398195:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177288.398197:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177288.398199:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177288.398203:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786177288.398206:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177288.398211:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177288.398218:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177288.398220:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177288.398224:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177288.398249:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177288.398252:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177288.398255:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177288.398257:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177288.398258:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177288.398260:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177288.398263:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11962 00000400:00000200:3.0:1786177288.398266:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177288.398269:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177288.398272:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177288.398274:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.398276:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177288.398285:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.398288:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177288.398290:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177288.398291:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177288.398293:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177288.398295:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177288.398296:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177288.398298:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177288.398302:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.398304:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177288.398305:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177288.398307:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177288.398309:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177288.398312:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177288.398314:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177288.398316:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177288.398319:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177288.398321:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.398323:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177288.398324:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177288.398327:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.398356:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177288.398374:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786177288.398376:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177288.398386:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177288.398388:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177288.398389:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177288.398391:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177288.398392:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177288.398396:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786177288.398399:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177288.398404:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177288.398416:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177288.398418:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177288.398422:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177288.398460:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177288.398465:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11972) 00000100:00000001:0.0:1786177288.398673:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.398675:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177288.398680:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177288.398697:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177288.398702:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177288.398704:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306635269 00000100:00000001:0.0:1786177288.398706:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306639105 00000100:00000001:0.0:1786177288.398708:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306635269 00000100:00000001:0.0:1786177288.398710:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306639105 00000100:00000001:0.0:1786177288.398712:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177288.398715:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.398716:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177288.398737:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177288/real 1786177288] req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177288.398752:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177288.398754:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177288.398756:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177288.398758:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177288.398760:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177288.398762:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177288.398765:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177288.398792:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.398798:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177288.398800:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.398802:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177288.398804:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177288.398807:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.398813:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177288.398816:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.398837:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177288.398839:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.398841:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177288.398842:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.398844:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177288.398846:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177288.398849:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177288.398852:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177288.398855:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177288.398859:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177288.398866:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598848:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177288.398871:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.398895:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177288.398899:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327800 x1872930235598720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.398905:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177288.398908:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177288.398910:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177288.398913:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177288.398915:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177288.398917:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177288.398919:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177288.398921:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:0.0:1786177288.398932:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177288.398933:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177288.398935:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327800. 00000100:00000001:0.0:1786177288.398938:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177288.398939:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.398941:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177288.398944:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326d80 x1872930235598848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.398950:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177288.398951:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177288.398953:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177288.398955:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177288.398956:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177288.398957:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177288.398958:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177288.398959:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:0.0:1786177288.399234:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177288.399236:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177288.399238:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326d80. 00000100:00000001:0.0:1786177288.399241:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177288.399242:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.399245:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.399251:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177288.399253:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.399255:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177288.399256:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177288.399258:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177288.399262:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399269:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177288.399270:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177288.399273:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177288.399274:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177288.399292:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177288.399296:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598976:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177288.399300:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177288.399302:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177288.399304:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177288.399305:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177288.399309:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177288.399311:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177288.399315:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177288.399318:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177288.399321:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177288.399324:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235598976, portal 10 00000100:00000001:0.0:1786177288.399326:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177288.399334:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280810880 : -110371428740736 : ffff9b9e26325180) 00000100:00000040:0.0:1786177288.399341:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399347:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177288.399349:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177288.399352:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177288.399355:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235598976, offset 0 00000400:00000200:0.0:1786177288.399375:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177288.399394:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177288.399398:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177288.399401:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177288.399404:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177288.399406:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177288.399408:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177288.399412:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426746:426746:256:4294967295] 192.168.202.132@tcp LPNI seq info [426708:426708:8:4294967295] 00000400:00000200:0.0:1786177288.399416:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177288.399422:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177288.399426:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177288.399440:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177288.399445:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177288.399448:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1786177288.399561:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177288.399564:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1786177288.399567:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177288.399569:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177288.399573:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177288.399587:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177288.399591:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177288.399592:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177288.399595:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177288.399598:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399606:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177288.399609:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399615:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.399616:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177288.399620:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177288.399623:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177288.399638:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177288.399640:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306635270 00000100:00000001:0.0:1786177288.399643:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306639105 00000100:00000001:0.0:1786177288.399645:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306635270 00000100:00000001:0.0:1786177288.399647:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306639105 00000100:00000001:0.0:1786177288.399649:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177288.399651:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.399652:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177288.399657:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177288/real 1786177288] req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177288.399662:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177288.399663:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177288.399664:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177288.399666:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177288.399667:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177288.399668:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177288.399670:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177288.399683:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399687:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177288.399688:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.399689:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177288.399690:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177288.399692:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399696:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177288.399699:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399704:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177288.399707:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.399708:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177288.399709:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.399711:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177288.399713:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177288.399716:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177288.399719:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177288.399721:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177288.399725:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177288.399744:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235598976:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177288.399748:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177288.399751:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399758:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177288.399759:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177288.399761:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177288.399762:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177288.399763:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177288.399765:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599104:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177288.399767:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177288.399769:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177288.399770:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177288.399771:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177288.399773:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177288.399774:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177288.399786:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177288.399789:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177288.399791:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177288.399793:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235599104, portal 4 00000100:00000001:0.0:1786177288.399794:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177288.399795:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280813568 : -110371428738048 : ffff9b9e26325c00) 00000100:00000040:0.0:1786177288.399800:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399803:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177288.399805:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177288.399806:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177288.399807:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235599104, offset 0 00000400:00000200:0.0:1786177288.399810:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177288.399812:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177288.399814:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177288.399816:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177288.399818:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177288.399819:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177288.399821:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177288.399823:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426747:426747:256:4294967295] 192.168.202.132@tcp LPNI seq info [426709:426709:8:4294967295] 00000400:00000200:0.0:1786177288.399825:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177288.399836:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177288.399839:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177288.399841:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177288.399843:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177288.399845:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1786177288.399847:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177288.399849:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1786177288.399851:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177288.399852:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177288.399854:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177288.399856:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177288.399858:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177288.399859:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177288.399861:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177288.399864:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399868:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177288.399870:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399876:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.399878:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177288.399890:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177288.399893:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177288.399896:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177288.399898:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306635270 00000100:00000001:0.0:1786177288.399901:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306639105 00000100:00000001:0.0:1786177288.399903:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306635270 00000100:00000001:0.0:1786177288.399905:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306639105 00000100:00000001:0.0:1786177288.399907:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177288.399909:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.399910:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177288.399915:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177288/real 1786177288] req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177288.399922:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177288.399923:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177288.399924:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177288.399925:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177288.399927:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177288.399928:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177288.399942:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177288.399947:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.399955:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177288.399956:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.399957:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177288.399958:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177288.400016:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400021:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177288.400024:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400028:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177288.400029:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.400050:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177288.400051:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.400052:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177288.400053:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177288.400055:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177288.400057:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177288.400058:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177288.400060:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177288.400065:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599104:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177288.400068:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177288.400069:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400073:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177288.400074:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177288.400075:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177288.400076:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177288.400078:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177288.400080:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599232:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177288.400089:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177288.400091:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177288.400093:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177288.400094:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177288.400098:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177288.400099:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177288.400103:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177288.400105:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177288.400109:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177288.400111:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235599232, portal 4 00000100:00000001:0.0:1786177288.400113:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177288.400114:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280815360 : -110371428736256 : ffff9b9e26326300) 00000100:00000040:0.0:1786177288.400120:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400126:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177288.400128:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177288.400130:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177288.400156:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235599232, offset 0 00000400:00000200:0.0:1786177288.400162:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177288.400167:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177288.400170:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177288.400173:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177288.400176:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177288.400178:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177288.400180:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177288.400270:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426748:426748:256:4294967295] 192.168.202.132@tcp LPNI seq info [426710:426710:8:4294967295] 00000400:00000200:0.0:1786177288.400275:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177288.400281:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177288.400296:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177288.400300:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177288.400304:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177288.400306:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1786177288.400310:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177288.400313:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1786177288.400316:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177288.400318:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177288.400321:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177288.400324:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177288.400327:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177288.400329:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177288.400331:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177288.400355:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400381:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177288.400384:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177313 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400402:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.400403:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177288.400407:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177288.400410:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177288.400412:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177288.400414:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306635273 00000100:00000001:0.0:1786177288.400417:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306639105 00000100:00000001:0.0:1786177288.400418:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306635273 00000100:00000001:0.0:1786177288.400420:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306639105 00000100:00000001:0.0:1786177288.400436:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177288.400444:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.400446:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177288.400458:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177288/real 1786177288] req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177288.400466:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177288.400467:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177288.400469:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177288.400472:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177288.400486:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177288.400488:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177288.400492:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177288.400499:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400506:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177288.400507:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.400509:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177288.400510:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177288.400514:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400521:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177288.400536:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400543:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177288.400545:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.400547:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177288.400548:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.400550:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177288.400551:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177288.400553:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177288.400556:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177288.400558:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177288.400562:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177288.400570:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599232:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177288.400589:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.400591:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177288.400595:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325180 x1872930235598976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400601:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177288.400603:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177288.400605:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177288.400607:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177288.400609:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177288.400610:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177288.400612:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177288.400614:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:0.0:1786177288.400617:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177288.400618:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177288.400619:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325180. 00000100:00000001:0.0:1786177288.400622:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177288.400623:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.400629:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177288.400633:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325c00 x1872930235599104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400639:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177288.400640:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177288.400642:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177288.400643:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177288.400644:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177288.400645:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177288.400647:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177288.400648:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:0.0:1786177288.400650:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177288.400651:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177288.400652:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325c00. 00000100:00000001:0.0:1786177288.400654:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177288.400656:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.400657:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177288.400660:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326300 x1872930235599232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177313 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177288.400665:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177288.400666:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177288.400668:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177288.400669:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177288.400670:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177288.400671:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177288.400672:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177288.400674:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 02000000:00000001:0.0:1786177288.400836:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177288.400838:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177288.400840:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326300. 00000100:00000001:0.0:1786177288.400842:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177288.400843:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.400845:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.400849:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177288.400850:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.400852:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177288.400854:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177288.908460:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177288.908467:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177288.908469:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177288.908475:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177288.908479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177288.908483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.908486:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177288.908489:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177288.908492:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177289.292536:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177289.292543:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177289.292545:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177289.292551:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177289.292555:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177289.292559:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177289.292561:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177289.292564:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177289.292567:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177289.292646:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177289.292650:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177289.292654:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177289.292656:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177289.292659:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177289.292661:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177289.292715:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177289.292719:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177289.292723:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177289.933172:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177289.933180:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177289.933182:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177289.933187:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177289.933190:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177289.933194:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177289.933196:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177289.933198:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177289.933201:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177290.316502:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177290.316508:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177290.316513:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177290.316515:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177290.316518:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177290.316519:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177290.316581:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177290.316585:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177290.316589:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177290.319536:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177290.319542:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177290.319544:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177290.319548:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177290.319552:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177290.319556:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177290.319558:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177290.319562:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177290.319564:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177290.956544:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177290.956554:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177290.956557:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177290.956564:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177290.956568:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177290.956572:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177290.956575:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177290.956577:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177290.956581:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177291.340486:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177291.340493:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177291.340498:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177291.340501:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177291.340504:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177291.340507:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177291.340510:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177291.340512:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177291.340514:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177291.340517:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177291.340520:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177291.340521:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177291.340625:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177291.340629:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177291.340631:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177291.340635:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177291.340638:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177291.340641:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177291.340643:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177291.340646:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177291.340649:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177291.340994:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177291.340998:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177291.341002:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177291.980507:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177291.980514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177291.980516:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177291.980522:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177291.980525:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177291.980529:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177291.980531:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177291.980534:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177291.980546:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177292.300476:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306639105, at 4306639168 00000100:00000040:1.0:1786177292.300476:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306639105, at 4306639168 00000100:00000010:0.0:1786177292.300491:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5840. 00000100:00000010:1.0:1786177292.300493:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5f00. 00010000:00000001:0.0:1786177292.364552:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177292.364560:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177292.364566:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177292.364568:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177292.364571:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177292.364573:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177292.365189:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177292.365193:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177292.365196:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177292.365200:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177292.365203:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177292.365206:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177292.365209:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177292.365211:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177292.365214:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177292.365467:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177292.365471:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177292.365474:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177293.004486:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177293.004493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177293.004496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177293.004500:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177293.004503:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177293.004506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177293.004509:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177293.004512:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177293.004514:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177293.388428:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177293.388436:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177293.388438:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177293.388444:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177293.388447:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177293.388451:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177293.388453:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177293.388457:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177293.388459:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177293.388530:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177293.388533:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177293.388537:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177293.388539:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177293.388542:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177293.388600:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177293.388721:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177293.388725:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177293.388729:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177293.516540:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177293.516550:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177293.516553:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177293.516556:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177293.516561:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177293.516568:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11967 00000400:00000200:2.0:1786177293.516576:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177293.516582:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177293.516585:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177293.516589:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.516592:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177293.516594:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.516596:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177293.516600:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177293.516602:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177293.516605:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177293.516608:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177293.516610:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177293.516613:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177293.516616:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.516618:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177293.516620:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.516622:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177293.516626:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177293.516628:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177293.516631:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177293.516634:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177293.516637:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177293.516640:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.516642:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177293.516645:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177293.516654:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734700. 00000020:00000040:2.0:1786177293.516658:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177293.516662:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177293.516664:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.516667:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177293.516669:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177293.516673:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177293.516680:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:2.0:1786177293.516686:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177293.516694:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177293.516704:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177293.516708:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177293.516713:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088313762] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.516783:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177293.516788:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177293.516791:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177293.516799:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.516808:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177293.516811:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.516811:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177293.516815:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1786177293.516817:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177293.516820:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177293.516821:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177293.516823:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1786177293.516830:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177293.516833:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177293.516838:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235599360:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177293.516841:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1786177293.516844:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1786177293.516848:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11967 00000400:00000200:2.0:1786177293.516853:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177293.516856:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 02000000:00000001:0.0:1786177293.516859:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1786177293.516860:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1786177293.516864:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177293.516865:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:0.0:1786177293.516868:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1786177293.516869:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177293.516872:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.516875:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000010:0.0:1786177293.516876:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 00000020:00000001:2.0:1786177293.516879:0:40394:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:0.0:1786177293.516881:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177293.516882:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177293.516886:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177293.516888:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000400:00000200:0.0:1786177293.516889:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:2.0:1786177293.516891:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000400:00000010:0.0:1786177293.516896:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000020:00000001:2.0:1786177293.516896:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177293.516902:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.516904:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177293.516906:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.516908:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177293.516910:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177293.516913:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000010:0.0:1786177293.516914:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000001:2.0:1786177293.516915:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177293.516920:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1786177293.516924:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235599360, portal 25 02000000:00000001:2.0:1786177293.516925:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177293.516930:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1786177293.516930:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.516933:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469617408 : -110371239934208 : ffff9b9e31734700) 00000100:00000001:2.0:1786177293.516935:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177293.516941:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000040:0.0:1786177293.516943:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1786177293.516946:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.516949:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177293.516955:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736d80. 00000100:00000001:0.0:1786177293.516957:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:2.0:1786177293.516959:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177293.516964:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177293.516966:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.516968:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177293.516969:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177293.516971:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177293.516975:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000040:0.0:1786177293.516978:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177293.516980:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1786177293.516986:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00100000:2.0:1786177293.516988:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1786177293.516990:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235599360, offset 0 00000400:00000200:0.0:1786177293.517001:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517001:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177293.517006:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177293.517012:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177293.517012:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e4f7686] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1786177293.517021:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:2.0:1786177293.517025:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177293.517029:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1786177293.517029:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1786177293.517035:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00080000:2.0:1786177293.517035:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177293.517039:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1786177293.517042:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1786177293.517042:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177293.517047:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1786177293.517048:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00080000:2.0:1786177293.517054:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11967 00000400:00000200:0.0:1786177293.517057:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426749:426749:256:4294967295] 192.168.202.132@tcp LPNI seq info [426711:426711:8:4294967295] 00000400:00000200:2.0:1786177293.517062:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177293.517068:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177293.517069:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177293.517073:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177293.517077:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517079:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177293.517083:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517087:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177293.517089:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177293.517090:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177293.517092:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177293.517093:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177293.517095:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:0.0:1786177293.517100:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000020:00000001:2.0:1786177293.517100:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177293.517107:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517111:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1786177293.517112:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000040:2.0:1786177293.517116:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000010:0.0:1786177293.517120:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000100:00000001:2.0:1786177293.517121:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177293.517125:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177293.517129:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:0.0:1786177293.517134:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177293.517134:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177293.517139:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.0:1786177293.517141:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 02000000:00000001:2.0:1786177293.517142:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177293.517145:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.517147:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177293.517149:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177293.517151:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.517152:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177293.517156:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737480. 00000020:00000040:2.0:1786177293.517159:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177293.517161:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177293.517162:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.517164:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177293.517165:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177293.517167:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177293.517170:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab400. 00000100:00000001:2.0:1786177293.517173:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177293.517177:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177293.517184:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177293.517186:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177293.517189:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046bba14a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177293.517196:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177293.517199:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177293.517202:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177293.517204:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177293.517205:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177293.517208:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177293.517210:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11967 00000400:00000200:2.0:1786177293.517213:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177293.517216:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177293.517218:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177293.517220:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517222:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177293.517224:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517226:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177293.517228:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177293.517229:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177293.517231:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177293.517232:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177293.517234:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177293.517236:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177293.517239:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517241:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177293.517243:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517244:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177293.517246:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177293.517249:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517251:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177293.517252:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177293.517255:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177293.517256:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.517258:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177293.517261:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177293.517263:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.517265:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177293.517268:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000020:00000040:2.0:1786177293.517270:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177293.517272:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177293.517273:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.517275:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177293.517276:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177293.517277:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177293.517280:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:2.0:1786177293.517283:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177293.517287:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177293.517293:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177293.517295:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177293.517298:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177293.517305:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177293.517308:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177293.517311:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177293.517313:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177293.517314:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177293.517317:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177293.517319:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11967 00000400:00000200:2.0:1786177293.517322:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177293.517324:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177293.517326:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177293.517328:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517330:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177293.517332:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517334:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177293.517336:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177293.517337:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177293.517339:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177293.517340:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177293.517342:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177293.517344:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177293.517346:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517348:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177293.517350:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517352:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177293.517354:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177293.517356:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177293.517385:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177293.517387:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177293.517390:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177293.517391:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.517393:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177293.517395:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177293.517397:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.517398:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177293.517410:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786177293.517413:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177293.517415:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177293.517417:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177293.517418:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177293.517420:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177293.517421:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177293.517425:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786177293.517427:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177293.517432:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177293.517444:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177293.517446:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177293.517449:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177293.517456:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177293.517460:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11978) 00000800:00000200:0.1:1786177293.517554:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177293.517557:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:0.0:1786177293.517562:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177293.517564:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177293.517571:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177293.517575:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177293.517580:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177293.517583:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177293.517587:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177293.517591:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.517600:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177293.517604:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.517610:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.517612:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177293.517618:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177293.517629:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177293.517632:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306644225 00000100:00000010:0.0:1786177293.517637:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5840. 00000100:00000001:0.0:1786177293.517643:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177293.517649:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306644225 00000100:00000010:0.0:1786177293.517652:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5d20. 00000100:00000001:0.0:1786177293.517655:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177293.517658:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177293.517661:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.517664:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177293.517668:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177293/real 1786177293] req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177293.517677:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177293.517678:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177293.517680:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177293.517682:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177293.517684:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177293.517687:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177293.517690:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177293.517698:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.517704:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177293.517705:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.517707:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177293.517709:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177293.517712:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.517718:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177293.517723:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.517729:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177293.517732:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.517733:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177293.517734:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.517736:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177293.517738:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177293.517740:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177293.517744:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177293.517747:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177293.517751:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177293.517759:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235599360:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177293.517764:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.517765:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177293.517768:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734700 x1872930235599360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.517774:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177293.517777:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177293.517779:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177293.517782:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177293.517784:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177293.517786:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177293.517788:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177293.517790:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:0.0:1786177293.517794:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177293.517795:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177293.517797:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734700. 00000100:00000001:0.0:1786177293.517799:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177293.517800:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.517803:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.517809:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177293.517811:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.517813:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177293.517822:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177293.517824:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177293.517827:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.517834:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177293.517836:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177293.517838:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177293.517839:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177293.517842:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177293.517845:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599488:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177293.517848:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177293.517850:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177293.517851:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177293.517853:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177293.517856:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177293.517858:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177293.517861:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177293.517864:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177293.517867:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177293.517870:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235599488, portal 10 00000100:00000001:0.0:1786177293.517872:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177293.517873:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469627264 : -110371239924352 : ffff9b9e31736d80) 00000100:00000040:0.0:1786177293.517880:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.517886:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177293.517888:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177293.517890:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177293.517893:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235599488, offset 0 00000400:00000200:0.0:1786177293.517897:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177293.517901:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177293.517904:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177293.517907:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177293.517910:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177293.517912:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177293.517914:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177293.517917:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426750:426750:256:4294967295] 192.168.202.132@tcp LPNI seq info [426712:426712:8:4294967295] 00000400:00000200:0.0:1786177293.517921:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177293.517927:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177293.517931:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177293.517934:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177293.517938:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177293.517941:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1786177293.517945:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177293.517948:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:0.0:1786177293.517952:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177293.517953:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177293.517956:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177293.517959:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177293.517962:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177293.517964:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177293.517966:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177293.517970:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.517977:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177293.517979:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.517984:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.517986:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177293.517988:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177293.517992:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177293.517995:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177293.517996:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306640388 00000100:00000001:0.0:1786177293.517999:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306644225 00000100:00000001:0.0:1786177293.518000:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306640388 00000100:00000001:0.0:1786177293.518002:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306644225 00000100:00000001:0.0:1786177293.518004:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177293.518006:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.518007:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177293.518011:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177293/real 1786177293] req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177293.518024:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177293.518025:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177293.518026:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177293.518028:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177293.518030:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177293.518031:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177293.518034:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177293.518040:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518046:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177293.518047:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518048:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177293.518049:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177293.518052:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518057:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177293.518060:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518066:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177293.518068:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518069:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177293.518070:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518071:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177293.518073:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177293.518075:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177293.518078:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177293.518080:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177293.518083:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177293.518100:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599488:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177293.518104:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177293.518106:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518112:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177293.518113:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177293.518116:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177293.518117:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177293.518119:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177293.518122:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599616:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177293.518125:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177293.518127:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177293.518128:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177293.518130:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177293.518133:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177293.518135:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177293.518138:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177293.518246:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177293.518252:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177293.518255:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235599616, portal 10 00000100:00000001:0.0:1786177293.518257:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177293.518258:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469629056 : -110371239922560 : ffff9b9e31737480) 00000100:00000040:0.0:1786177293.518263:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518267:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177293.518268:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177293.518270:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177293.518272:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235599616, offset 0 00000400:00000200:0.0:1786177293.518274:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177293.518277:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177293.518280:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177293.518281:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177293.518283:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177293.518285:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177293.518286:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177293.518288:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426751:426751:256:4294967295] 192.168.202.132@tcp LPNI seq info [426713:426713:8:4294967295] 00000400:00000200:0.0:1786177293.518290:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177293.518294:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177293.518296:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177293.518298:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177293.518300:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177293.518302:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1786177293.518305:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177293.518306:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:0.0:1786177293.518308:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177293.518310:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177293.518312:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177293.518314:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177293.518316:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177293.518317:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177293.518325:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177293.518329:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518336:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177293.518338:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518344:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.518346:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177293.518349:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177293.518352:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177293.518355:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177293.518375:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306640389 00000100:00000001:0.0:1786177293.518377:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306644225 00000100:00000001:0.0:1786177293.518378:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306640389 00000100:00000001:0.0:1786177293.518379:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306644225 00000100:00000001:0.0:1786177293.518381:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177293.518382:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.518383:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177293.518386:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177293/real 1786177293] req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177293.518391:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177293.518392:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177293.518393:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177293.518395:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177293.518396:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177293.518397:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177293.518399:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177293.518424:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518429:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177293.518430:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518431:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177293.518432:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177293.518434:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518438:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177293.518440:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518444:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177293.518445:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518446:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177293.518447:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518448:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177293.518450:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177293.518451:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177293.518453:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177293.518455:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177293.518458:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177293.518465:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599616:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177293.518468:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177293.518471:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518477:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177293.518478:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177293.518480:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177293.518482:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177293.518484:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177293.518487:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599744:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177293.518490:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177293.518492:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177293.518494:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177293.518495:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177293.518498:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177293.518500:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177293.518503:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177293.518505:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177293.518507:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177293.518509:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235599744, portal 4 00000100:00000001:0.0:1786177293.518511:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177293.518512:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469621888 : -110371239929728 : ffff9b9e31735880) 00000100:00000040:0.0:1786177293.518516:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518519:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177293.518521:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177293.518522:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177293.518523:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235599744, offset 0 00000400:00000200:0.0:1786177293.518526:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177293.518528:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177293.518530:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177293.518532:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177293.518533:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177293.518535:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177293.518536:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177293.518538:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426752:426752:256:4294967295] 192.168.202.132@tcp LPNI seq info [426714:426714:8:4294967295] 00000400:00000200:0.0:1786177293.518541:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177293.518544:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177293.518546:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177293.518548:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177293.518550:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177293.518552:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1786177293.518554:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177293.518556:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:0.0:1786177293.518558:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177293.518559:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177293.518561:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177293.518563:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177293.518565:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177293.518566:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177293.518568:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177293.518569:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518578:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177293.518579:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518583:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.518584:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177293.518586:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177293.518588:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177293.518589:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177293.518590:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306640389 00000100:00000001:0.0:1786177293.518592:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306644225 00000100:00000001:0.0:1786177293.518593:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306640389 00000100:00000001:0.0:1786177293.518594:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306644225 00000100:00000001:0.0:1786177293.518595:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177293.518596:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.518597:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177293.518599:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177293/real 1786177293] req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177293.518603:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177293.518604:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177293.518605:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177293.518606:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177293.518607:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177293.518608:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177293.518610:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177293.518615:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518618:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177293.518619:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518621:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177293.518621:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177293.518623:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518627:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177293.518629:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518633:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177293.518634:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518635:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177293.518635:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518636:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177293.518637:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177293.518639:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177293.518640:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177293.518641:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177293.518643:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177293.518647:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599744:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177293.518649:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177293.518651:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518655:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177293.518656:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177293.518657:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177293.518658:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177293.518659:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177293.518661:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599872:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177293.518663:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177293.518664:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177293.518665:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177293.518666:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177293.518667:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177293.518668:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177293.518670:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177293.518671:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177293.518673:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177293.518674:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235599872, portal 4 00000100:00000001:0.0:1786177293.518675:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177293.518676:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469622784 : -110371239928832 : ffff9b9e31735c00) 00000100:00000040:0.0:1786177293.518679:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518683:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177293.518684:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177293.518685:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177293.518686:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235599872, offset 0 00000400:00000200:0.0:1786177293.518688:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177293.518690:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177293.518692:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177293.518694:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177293.518695:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177293.518696:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177293.518698:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177293.518699:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426753:426753:256:4294967295] 192.168.202.132@tcp LPNI seq info [426715:426715:8:4294967295] 00000400:00000200:0.0:1786177293.518701:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177293.518705:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177293.518707:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177293.518708:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177293.518710:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177293.518712:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1786177293.518714:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177293.518716:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:0.0:1786177293.518718:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177293.518719:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177293.518721:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177293.518723:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177293.518724:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177293.518725:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177293.518726:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177293.518728:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518732:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177293.518734:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177318 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518737:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.518738:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177293.518740:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177293.518741:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177293.518743:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177293.518744:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306640391 00000100:00000001:0.0:1786177293.518746:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306644225 00000100:00000001:0.0:1786177293.518747:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306640391 00000100:00000001:0.0:1786177293.518748:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306644225 00000100:00000001:0.0:1786177293.518749:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177293.518750:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.518751:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177293.518753:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177293/real 1786177293] req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177293.518757:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177293.518758:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177293.518759:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177293.518760:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177293.518761:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177293.518762:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177293.518764:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177293.518768:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518771:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177293.518772:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518773:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177293.518773:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177293.518775:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518778:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177293.518780:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518784:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177293.518785:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518786:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177293.518786:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518787:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177293.518788:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177293.518789:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177293.518791:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177293.518792:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177293.518794:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177293.518802:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235599872:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177293.518804:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518805:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177293.518807:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736d80 x1872930235599488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518810:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177293.518811:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177293.518812:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177293.518813:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177293.518814:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177293.518815:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177293.518816:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177293.518817:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:0.0:1786177293.518819:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177293.518819:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177293.518821:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736d80. 00000100:00000001:0.0:1786177293.518823:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177293.518823:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518825:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177293.518826:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737480 x1872930235599616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518830:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177293.518830:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177293.518831:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177293.518832:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177293.518833:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177293.518834:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177293.518836:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177293.518837:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:0.0:1786177293.518839:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177293.518840:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177293.518841:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737480. 00000100:00000001:0.0:1786177293.518843:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177293.518844:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518846:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177293.518849:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735880 x1872930235599744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518854:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177293.518856:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177293.518857:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177293.518859:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177293.518860:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177293.518861:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177293.518863:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177293.518864:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:0.0:1786177293.518866:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177293.518868:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177293.518869:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735880. 00000100:00000001:0.0:1786177293.518871:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177293.518872:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518874:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177293.518877:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735c00 x1872930235599872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177318 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177293.518883:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177293.518884:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177293.518885:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177293.518886:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177293.518887:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177293.518887:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177293.518888:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177293.518889:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:0.0:1786177293.518890:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177293.518891:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177293.518892:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735c00. 00000100:00000001:0.0:1786177293.518893:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177293.518893:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518894:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177293.518897:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177293.518898:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177293.518899:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177293.518900:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177294.028514:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177294.028522:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177294.028525:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177294.028531:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177294.028534:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177294.028537:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177294.028540:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177294.028542:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177294.028545:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177294.412523:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177294.412526:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177294.412534:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177294.412534:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177294.412538:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177294.412539:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177294.412558:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177294.412563:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177294.412565:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177294.412566:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177294.412569:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177294.412573:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177294.412575:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177294.412580:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177294.412592:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177294.412594:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177294.412596:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177294.412598:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177294.412599:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177294.412600:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177294.412602:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177294.412623:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177294.412630:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177294.412634:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177294.412638:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177294.412641:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177294.412645:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177294.412957:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177294.412961:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177294.412964:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177295.053506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177295.053513:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177295.053516:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177295.053520:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177295.053524:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177295.053528:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177295.053530:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177295.053533:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177295.053535:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177295.436447:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177295.436456:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177295.436458:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177295.436458:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177295.436463:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177295.436469:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177295.436471:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177295.436473:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177295.436476:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177295.436478:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177295.436479:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177295.436482:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177295.436487:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177295.436489:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177295.436491:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177295.436587:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177295.436590:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177295.436593:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177296.077279:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177296.077287:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177296.077289:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177296.077294:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177296.077298:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177296.077301:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177296.077303:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177296.077306:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177296.077309:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177296.460483:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177296.460491:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177296.460497:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177296.460499:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177296.460512:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177296.460514:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177296.460573:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177296.460592:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177296.460594:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177296.460598:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177296.460613:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177296.460616:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177296.460618:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177296.460621:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177296.460624:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177296.461424:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177296.461428:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177296.461431:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177297.100559:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177297.100565:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177297.100566:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177297.100571:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177297.100574:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177297.100577:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177297.100579:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177297.100582:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177297.100584:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177297.420498:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306644225, at 4306644288 00000100:00000010:0.0:1786177297.420510:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5840. 00000100:00000040:1.0:1786177297.420571:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306644225, at 4306644288 00000100:00000010:1.0:1786177297.420575:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5d20. 00000100:00000001:1.0:1786177297.484479:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177297.484487:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177297.484489:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177297.484494:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177297.484498:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177297.484502:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177297.484505:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177297.484508:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177297.484510:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177297.484608:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177297.484611:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177297.484615:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177297.484617:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177297.484620:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177297.484630:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177297.484693:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177297.484697:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177297.484700:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177298.124672:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177298.124683:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177298.124685:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177298.124693:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177298.124697:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177298.124702:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.124705:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177298.124708:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.124711:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177298.508518:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177298.508524:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177298.508527:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177298.508531:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177298.508533:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177298.508534:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177298.508535:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177298.508538:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177298.508539:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177298.508541:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177298.508542:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177298.508547:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177298.508550:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177298.508552:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177298.508556:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177298.508611:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177298.508617:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177298.508621:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177298.637614:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177298.637628:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177298.637631:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177298.637635:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177298.637639:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177298.637647:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11973 00000400:00000200:2.0:1786177298.637655:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177298.637661:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177298.637663:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177298.637665:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637668:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177298.637670:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637672:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177298.637675:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177298.637677:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177298.637679:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177298.637682:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177298.637683:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177298.637685:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177298.637686:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637688:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177298.637689:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637690:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177298.637693:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177298.637695:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637697:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177298.637699:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177298.637702:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177298.637704:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.637707:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177298.637709:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177298.637715:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786177298.637718:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177298.637722:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177298.637723:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.637726:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177298.637729:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177298.637733:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177298.637744:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786177298.637752:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177298.637765:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177298.637777:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177298.637782:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177298.637787:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177298.637804:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177298.637807:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177298.637810:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177298.637812:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177298.637813:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177298.637816:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177298.637819:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11973 00000400:00000200:2.0:1786177298.637823:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177298.637826:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177298.637829:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177298.637831:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177298.637832:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177298.637833:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177298.637835:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637837:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177298.637838:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1786177298.637840:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1786177298.637841:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1786177298.637841:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177298.637843:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177298.637845:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177298.637847:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786177298.637848:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177298.637849:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177298.637852:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637855:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1786177298.637857:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1786177298.637857:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637858:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177298.637859:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177298.637861:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 02000000:00000001:0.0:1786177298.637863:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1786177298.637863:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637865:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177298.637866:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177298.637867:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177298.637873:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177298.637875:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.637877:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:0.0:1786177298.637879:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1786177298.637881:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1786177298.637882:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235600000:192.168.202.132@tcp:250:kworker.0 00000008:00000001:2.0:1786177298.637884:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.637887:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1786177298.637887:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177298.637890:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 02000000:00000001:0.0:1786177298.637892:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1786177298.637893:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786177298.637894:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177298.637895:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177298.637897:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1786177298.637897:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.637898:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177298.637899:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177298.637901:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177298.637903:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177298.637905:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177298.637905:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:2.0:1786177298.637908:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177298.637912:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:2.0:1786177298.637913:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1786177298.637916:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000100:00000001:2.0:1786177298.637920:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177298.637922:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1786177298.637924:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000040:2.0:1786177298.637926:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177298.637932:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235600000, portal 25 00000100:00000001:2.0:1786177298.637932:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.637935:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1786177298.637935:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1786177298.637937:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620992 : -110371239930624 : ffff9b9e31735500) 00000100:00080000:2.0:1786177298.637938:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177298.637939:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177298.637941:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1786177298.637942:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177298.637944:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177298.637946:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11973 00000400:00000200:2.0:1786177298.637949:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177298.637952:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177298.637955:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177298.637958:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177298.637959:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1786177298.637960:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177298.637962:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000040:2.0:1786177298.637962:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637964:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:0.0:1786177298.637965:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000020:00000001:2.0:1786177298.637966:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000200:0.0:1786177298.637968:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235600000, offset 0 00000020:00000040:2.0:1786177298.637968:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177298.637969:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177298.637971:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177298.637973:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:0.0:1786177298.637975:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000020:00000001:2.0:1786177298.637975:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177298.637977:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637979:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177298.637981:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637982:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:0.0:1786177298.637984:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:2.0:1786177298.637985:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177298.637987:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177298.637990:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177298.637991:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 02000000:00000001:2.0:1786177298.637991:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177298.637994:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1786177298.637996:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1786177298.637996:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.637997:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177298.637999:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1786177298.638001:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000008:00000001:2.0:1786177298.638001:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.638003:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177298.638005:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000010:2.0:1786177298.638006:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000400:00000200:0.0:1786177298.638008:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000040:2.0:1786177298.638008:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177298.638010:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177298.638012:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177298.638013:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426754:426754:256:4294967295] 192.168.202.132@tcp LPNI seq info [426716:426716:8:4294967295] 00000100:00000001:2.0:1786177298.638013:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177298.638015:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177298.638017:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177298.638018:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:2.0:1786177298.638020:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab400. 00000100:00000001:2.0:1786177298.638023:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177298.638026:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1786177298.638028:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177298.638033:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:0.0:1786177298.638035:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177298.638035:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177298.638039:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177298.638046:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177298.638049:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:0.0:1786177298.638050:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000100:00080000:2.0:1786177298.638052:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177298.638053:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177298.638054:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:0.0:1786177298.638057:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177298.638057:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177298.638060:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11973 00000800:00000200:0.0:1786177298.638061:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000400:00000200:2.0:1786177298.638063:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177298.638066:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000200:0.1:1786177298.638067:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:2.0:1786177298.638068:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177298.638070:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:0.1:1786177298.638071:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000100:00000001:2.0:1786177298.638073:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177298.638074:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000100:0.0:1786177298.638077:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177298.638077:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000010:0.0:1786177298.638079:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000020:00000001:2.0:1786177298.638079:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177298.638080:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177298.638082:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177298.638083:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000400:00000200:0.0:1786177298.638084:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:2.0:1786177298.638085:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177298.638087:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000200:0.0:1786177298.638088:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1786177298.638090:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.638092:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177298.638093:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:0.0:1786177298.638094:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000100:00000001:2.0:1786177298.638095:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:0.0:1786177298.638097:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000020:00000040:2.0:1786177298.638097:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177298.638099:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177298.638101:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1786177298.638101:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177298.638103:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1786177298.638105:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177298.638105:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177298.638107:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.638110:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177298.638112:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1786177298.638114:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1786177298.638114:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.638116:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177298.638117:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1786177298.638119:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000020:00000040:2.0:1786177298.638121:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177298.638124:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1786177298.638126:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177298.638126:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.638127:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1786177298.638128:0:2392:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1786177298.638129:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177298.638131:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177298.638134:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:0.0:1786177298.638135:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.638136:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177298.638139:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:2.0:1786177298.638140:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177298.638142:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306649345 00000100:00000010:0.0:1786177298.638146:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5840. 00000100:00000001:2.0:1786177298.638146:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177298.638148:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177298.638151:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638153:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177298.638158:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.638160:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306649345 00000100:00080000:2.0:1786177298.638160:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:0.0:1786177298.638163:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5c60. 00000100:00080000:2.0:1786177298.638163:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177298.638165:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177298.638166:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177298.638167:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177298.638168:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1786177298.638171:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1786177298.638171:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11973 00000100:00000001:0.0:1786177298.638173:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177298.638174:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177298.638177:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1786177298.638177:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177298.638178:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1786177298.638181:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177298/real 1786177298] req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177298.638181:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.638183:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177298.638185:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.638187:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177298.638189:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177298.638190:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000400:00000001:0.0:1786177298.638191:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1786177298.638191:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000001:0.0:1786177298.638192:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:2.0:1786177298.638193:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177298.638194:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000010:0.0:1786177298.638195:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000020:00000001:2.0:1786177298.638197:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000200:0.0:1786177298.638198:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177298.638199:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000040:2.0:1786177298.638201:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177298.638202:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1786177298.638202:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1786177298.638205:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177298.638205:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177298.638207:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177298.638209:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177298.638211:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177298.638213:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1786177298.638214:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177298.638214:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177298.638217:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177298.638218:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.638220:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177298.638221:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00100000:2.0:1786177298.638222:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1786177298.638223:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1786177298.638224:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.638225:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177298.638226:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.638228:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1786177298.638228:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 00000020:00000040:2.0:1786177298.638231:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00100000:0.0:1786177298.638232:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177298.638233:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177298.638234:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177298.638236:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177298.638237:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1786177298.638239:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1786177298.638239:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177298.638242:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000040:0.0:1786177298.638244:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177298.638245:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177298.638249:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177298.638252:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177298.638255:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177298.638255:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177298.638257:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177298.638257:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.638258:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.638261:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:2.0:1786177298.638261:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638263:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177298.638266:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1786177298.638269:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177298.638270:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177298.638272:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177298.638273:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11983) 00000100:00000040:0.0:1786177298.638276:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177298.638285:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235600000:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177298.638290:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.638292:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177298.638295:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735500 x1872930235600000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638303:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177298.638306:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177298.638308:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177298.638312:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177298.638313:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177298.638315:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177298.638317:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177298.638320:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:0.0:1786177298.638323:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177298.638325:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177298.638327:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735500. 00000100:00000001:0.0:1786177298.638330:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177298.638332:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.638334:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.638342:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177298.638345:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.638347:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177298.638348:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177298.638350:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177298.638471:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638556:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177298.638558:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177298.638561:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177298.638563:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177298.638565:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177298.638569:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600128:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177298.638580:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177298.638583:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177298.638585:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177298.638586:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177298.638590:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177298.638592:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177298.638596:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177298.638599:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177298.638603:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177298.638606:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235600128, portal 10 00000100:00000001:0.0:1786177298.638608:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177298.638610:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469630848 : -110371239920768 : ffff9b9e31737b80) 00000100:00000040:0.0:1786177298.638618:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638625:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177298.638628:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177298.638630:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177298.638633:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235600128, offset 0 00000400:00000200:0.0:1786177298.638637:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177298.638641:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177298.638644:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177298.638647:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177298.638649:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177298.638652:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177298.638654:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177298.638657:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426755:426755:256:4294967295] 192.168.202.132@tcp LPNI seq info [426717:426717:8:4294967295] 00000400:00000200:0.0:1786177298.638661:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177298.638667:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177298.638671:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177298.638675:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177298.638679:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177298.638682:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1786177298.638686:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177298.638688:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1786177298.638692:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177298.638693:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177298.638696:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177298.638700:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177298.638704:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177298.638705:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177298.638708:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177298.638712:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638720:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177298.638723:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638728:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.638730:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177298.638742:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177298.638746:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177298.638749:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177298.638751:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306645509 00000100:00000001:0.0:1786177298.638754:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306649345 00000100:00000001:0.0:1786177298.638756:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306645509 00000100:00000001:0.0:1786177298.638758:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306649345 00000100:00000001:0.0:1786177298.638760:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177298.638762:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.638764:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177298.638768:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177298/real 1786177298] req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177298.638776:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177298.638777:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177298.638779:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177298.638781:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177298.638783:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177298.638785:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177298.638788:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177298.638796:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638802:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177298.638804:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.638806:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177298.638807:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177298.638811:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638817:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177298.638821:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638828:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177298.638830:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.638832:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177298.638833:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.638835:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177298.638837:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177298.638840:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177298.638843:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177298.638845:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177298.638849:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177298.638856:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600128:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177298.638859:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177298.638861:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638865:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177298.638866:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177298.638867:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177298.638868:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177298.638869:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177298.638871:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600256:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177298.638873:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177298.638875:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177298.638876:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177298.638877:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177298.638880:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177298.638883:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177298.638886:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177298.638888:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177298.638892:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177298.638895:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235600256, portal 10 00000100:00000001:0.0:1786177298.638896:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177298.638898:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469616512 : -110371239935104 : ffff9b9e31734380) 00000100:00000040:0.0:1786177298.638904:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638910:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177298.638913:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177298.638914:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177298.638917:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235600256, offset 0 00000400:00000200:0.0:1786177298.638920:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177298.638924:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177298.638927:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177298.638930:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177298.638932:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177298.638934:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177298.638936:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177298.638939:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426756:426756:256:4294967295] 192.168.202.132@tcp LPNI seq info [426718:426718:8:4294967295] 00000400:00000200:0.0:1786177298.638943:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177298.638948:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177298.638952:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177298.638955:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177298.638959:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177298.638961:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1786177298.638965:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177298.638967:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1786177298.638971:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177298.638972:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177298.638975:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177298.638978:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177298.638981:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177298.638983:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177298.638985:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177298.638989:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.638995:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177298.638998:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639004:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.639006:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177298.639010:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177298.639012:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177298.639016:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177298.639018:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306645509 00000100:00000001:0.0:1786177298.639020:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306649345 00000100:00000001:0.0:1786177298.639023:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306645509 00000100:00000001:0.0:1786177298.639025:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306649345 00000100:00000001:0.0:1786177298.639027:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177298.639029:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.639031:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177298.639034:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177298/real 1786177298] req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177298.639050:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177298.639051:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177298.639053:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177298.639056:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177298.639057:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177298.639059:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177298.639062:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177298.639068:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639073:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177298.639074:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639075:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177298.639076:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177298.639078:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639082:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177298.639084:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639089:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177298.639090:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639091:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177298.639092:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639093:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177298.639094:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177298.639096:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177298.639097:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177298.639098:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177298.639100:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177298.639105:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600256:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177298.639108:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177298.639110:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639116:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177298.639117:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177298.639119:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177298.639125:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177298.639127:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177298.639129:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600384:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177298.639134:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177298.639136:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177298.639137:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177298.639139:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177298.639142:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177298.639143:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177298.639147:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177298.639150:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177298.639153:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177298.639156:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235600384, portal 4 00000100:00000001:0.0:1786177298.639158:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177298.639159:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469624576 : -110371239927040 : ffff9b9e31736300) 00000100:00000040:0.0:1786177298.639164:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639170:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177298.639172:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177298.639174:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177298.639176:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235600384, offset 0 00000400:00000200:0.0:1786177298.639181:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177298.639185:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177298.639188:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177298.639191:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177298.639193:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177298.639195:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177298.639197:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177298.639200:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426757:426757:256:4294967295] 192.168.202.132@tcp LPNI seq info [426719:426719:8:4294967295] 00000400:00000200:0.0:1786177298.639203:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177298.639208:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177298.639211:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177298.639214:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177298.639217:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177298.639219:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1786177298.639223:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177298.639225:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1786177298.639228:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177298.639230:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177298.639233:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177298.639236:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177298.639238:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177298.639241:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177298.639243:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177298.639246:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639253:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177298.639255:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639261:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.639263:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177298.639266:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177298.639269:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177298.639272:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177298.639274:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306645509 00000100:00000001:0.0:1786177298.639276:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306649345 00000100:00000001:0.0:1786177298.639277:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306645509 00000100:00000001:0.0:1786177298.639280:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306649345 00000100:00000001:0.0:1786177298.639281:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177298.639283:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.639285:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177298.639288:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177298/real 1786177298] req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177298.639295:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177298.639296:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177298.639298:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177298.639299:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177298.639301:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177298.639302:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177298.639305:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177298.639312:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639317:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177298.639319:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639320:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177298.639322:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177298.639324:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639330:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177298.639333:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639339:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177298.639341:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639342:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177298.639343:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639345:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177298.639346:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177298.639349:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177298.639386:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177298.639389:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177298.639394:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177298.639402:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600384:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177298.639406:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177298.639409:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639415:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177298.639416:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177298.639419:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177298.639421:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177298.639423:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177298.639426:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600512:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177298.639429:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177298.639431:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177298.639432:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177298.639434:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177298.639437:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177298.639439:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177298.639442:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177298.639444:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177298.639447:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177298.639450:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235600512, portal 4 00000100:00000001:0.0:1786177298.639452:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177298.639453:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469615616 : -110371239936000 : ffff9b9e31734000) 00000100:00000040:0.0:1786177298.639458:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639464:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177298.639466:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177298.639468:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177298.639470:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235600512, offset 0 00000400:00000200:0.0:1786177298.639474:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177298.639478:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177298.639490:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177298.639493:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177298.639495:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177298.639498:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177298.639500:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177298.639503:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426758:426758:256:4294967295] 192.168.202.132@tcp LPNI seq info [426720:426720:8:4294967295] 00000400:00000200:0.0:1786177298.639506:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177298.639512:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177298.639517:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177298.639520:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177298.639524:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177298.639526:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1786177298.639530:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177298.639533:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1786177298.639536:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177298.639538:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177298.639541:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177298.639544:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177298.639547:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177298.639548:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177298.639551:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177298.639554:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639561:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177298.639564:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177323 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639569:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.639570:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177298.639573:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177298.639576:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177298.639579:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177298.639581:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306645512 00000100:00000001:0.0:1786177298.639583:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306649345 00000100:00000001:0.0:1786177298.639585:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306645512 00000100:00000001:0.0:1786177298.639587:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306649345 00000100:00000001:0.0:1786177298.639589:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177298.639591:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.639593:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177298.639597:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177298/real 1786177298] req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177298.639606:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177298.639607:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177298.639609:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:0.0:1786177298.639612:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:0.0:1786177298.639613:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:0.0:1786177298.639615:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177298.639618:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177298.639626:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639632:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177298.639634:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639636:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177298.639637:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177298.639641:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639648:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177298.639652:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639658:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177298.639661:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639662:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177298.639664:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639666:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177298.639667:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177298.639671:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177298.639673:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177298.639676:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177298.639680:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177298.639687:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600512:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177298.639690:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639692:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177298.639694:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737b80 x1872930235600128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639700:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177298.639702:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177298.639704:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177298.639707:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177298.639708:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177298.639710:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177298.639712:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177298.639714:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:0.0:1786177298.639716:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177298.639718:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177298.639719:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737b80. 00000100:00000001:0.0:1786177298.639721:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177298.639723:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639724:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177298.639727:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734380 x1872930235600256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639733:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177298.639735:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177298.639737:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177298.639739:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177298.639740:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177298.639741:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177298.639744:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177298.639745:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:0.0:1786177298.639747:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177298.639749:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177298.639751:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734380. 00000100:00000001:0.0:1786177298.639753:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177298.639755:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639757:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177298.639762:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736300 x1872930235600384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639768:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177298.639769:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177298.639770:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177298.639772:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177298.639774:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177298.639775:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177298.639777:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177298.639779:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:0.0:1786177298.639781:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177298.639782:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177298.639784:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736300. 00000100:00000001:0.0:1786177298.639786:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177298.639787:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639788:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177298.639790:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734000 x1872930235600512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177323 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177298.639796:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177298.639797:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177298.639798:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177298.639800:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177298.639801:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177298.639802:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177298.639804:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177298.639805:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:0.0:1786177298.639807:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177298.639808:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177298.639809:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734000. 00000100:00000001:0.0:1786177298.639811:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177298.639812:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639814:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177298.639820:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177298.639822:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177298.639824:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177298.639825:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177299.148656:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177299.148668:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177299.148672:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177299.148679:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177299.148683:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177299.148688:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177299.148692:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177299.148695:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177299.148699:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177299.532527:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177299.532537:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177299.532544:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177299.532545:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177299.532549:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177299.532550:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177299.532786:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177299.532792:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177299.532796:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177299.532868:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177299.532873:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177299.532875:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177299.532878:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177299.532882:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177299.532885:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177299.532887:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177299.532889:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177299.532892:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177300.172592:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177300.172603:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177300.172605:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177300.172612:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177300.172616:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177300.172621:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177300.172624:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177300.172627:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177300.172631:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177300.556599:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177300.556610:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177300.556612:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177300.556620:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177300.556624:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177300.556629:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177300.556632:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177300.556636:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177300.556639:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177300.557124:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177300.557129:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177300.557135:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177300.557139:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177300.557142:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177300.557146:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177300.557150:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177300.557152:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177300.557155:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177300.557158:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177300.557162:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177300.557163:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177300.557462:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177300.557467:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177300.557470:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177301.196521:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177301.196532:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177301.196535:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177301.196544:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177301.196549:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177301.196553:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177301.196557:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177301.196560:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177301.196564:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177301.580524:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177301.580529:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177301.580534:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177301.580534:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177301.580537:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177301.580541:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177301.580542:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177301.580544:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177301.580546:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177301.580547:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177301.580548:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177301.580550:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177301.580553:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177301.580556:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177301.580560:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177301.580897:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177301.580902:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177301.580906:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177302.221005:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177302.221013:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177302.221016:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177302.221023:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177302.221027:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177302.221031:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177302.221034:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177302.221037:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177302.221040:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177302.540512:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306649345, at 4306649408 00000100:00000010:1.0:1786177302.540532:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5c60. 00000100:00000040:0.0:1786177302.540643:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306649345, at 4306649408 00000100:00000010:0.0:1786177302.540651:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5840. 00000100:00000001:1.0:1786177302.604508:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177302.604519:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177302.604521:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177302.604529:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177302.604534:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177302.604541:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177302.604544:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177302.604547:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177302.604550:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177302.604631:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177302.604636:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177302.604640:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177302.604641:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177302.604645:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177302.604646:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177302.604798:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177302.604804:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177302.604807:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177303.244522:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177303.244530:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177303.244534:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177303.244539:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177303.244543:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177303.244548:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177303.244551:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177303.244554:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177303.244557:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177303.628473:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177303.628484:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177303.628486:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177303.628494:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177303.628499:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177303.628503:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177303.628507:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177303.628510:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177303.628514:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177303.628597:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177303.628606:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177303.628609:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177303.628612:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177303.628617:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177303.628619:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177303.628621:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177303.628623:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177303.628627:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177303.628628:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177303.628632:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177303.628634:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177303.628635:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177303.628636:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177303.628638:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177303.628639:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177303.628641:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177303.628643:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177303.628717:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177303.628723:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177303.628726:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177303.756552:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177303.756562:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177303.756565:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177303.756569:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177303.756573:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177303.756580:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11978 00000400:00000200:3.0:1786177303.756588:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177303.756594:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177303.756597:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177303.756599:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756602:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177303.756604:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756606:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177303.756610:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177303.756611:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177303.756614:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177303.756617:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177303.756619:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177303.756620:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177303.756622:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756623:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177303.756625:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756626:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177303.756628:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177303.756630:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756631:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177303.756633:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177303.756636:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177303.756638:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.756640:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177303.756642:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177303.756652:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786177303.756655:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177303.756658:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177303.756659:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.756662:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177303.756665:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177303.756669:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177303.756677:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786177303.756684:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177303.756693:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177303.756700:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177303.756704:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177303.756707:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177303.756722:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177303.756725:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177303.756728:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177303.756729:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177303.756730:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177303.756732:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177303.756733:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11978 00000400:00000200:3.0:1786177303.756735:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177303.756737:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177303.756738:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177303.756739:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756740:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177303.756741:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756743:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177303.756744:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177303.756745:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177303.756746:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177303.756748:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177303.756749:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177303.756750:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177303.756752:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756753:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177303.756754:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756755:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177303.756756:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177303.756758:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756760:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177303.756761:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177303.756766:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177303.756767:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.756769:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177303.756774:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177303.756776:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.756777:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177303.756780:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786177303.756781:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177303.756783:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177303.756784:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.756786:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177303.756787:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177303.756789:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177303.756793:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786177303.756797:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177303.756802:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177303.756809:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177303.756811:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177303.756816:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177303.756824:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177303.756827:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177303.756831:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177303.756832:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177303.756834:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177303.756836:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177303.756839:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11978 00000400:00000200:3.0:1786177303.756843:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177303.756846:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177303.756848:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177303.756851:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756852:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177303.756854:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756858:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177303.756860:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177303.756861:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177303.756863:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177303.756866:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177303.756868:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177303.756871:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177303.756874:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756876:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177303.756880:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756882:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177303.756884:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177303.756887:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756889:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177303.756891:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177303.756894:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177303.756896:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.756897:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177303.756900:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177303.756902:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.756904:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177303.756907:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786177303.756909:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177303.756912:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177303.756913:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.756914:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177303.756916:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177303.756917:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177303.756921:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786177303.756934:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177303.756939:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177303.756953:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177303.756955:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177303.756959:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177303.756967:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177303.756970:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177303.756974:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177303.756976:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177303.756977:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177303.756981:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177303.756983:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11978 00000400:00000200:3.0:1786177303.756987:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177303.756990:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177303.756993:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177303.756996:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.756998:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177303.757000:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.757002:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177303.757005:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177303.757006:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177303.757008:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177303.757010:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177303.757013:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177303.757015:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177303.757018:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.757020:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177303.757022:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.757024:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177303.757027:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177303.757029:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177303.757032:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177303.757034:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177303.757037:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177303.757039:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.757041:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177303.757044:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177303.757046:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.757048:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177303.757052:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786177303.757054:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177303.757056:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177303.757058:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.757060:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177303.757061:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177303.757063:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177303.757066:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786177303.757069:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177303.757073:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177303.757079:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177303.757081:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177303.757084:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177303.757091:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177303.757093:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177303.757096:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177303.757099:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177303.757100:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177303.757103:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177303.757106:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11978 00000400:00000200:3.0:1786177303.757111:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177303.757115:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177303.757117:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177303.757121:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.757129:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177303.757132:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.757134:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177303.757137:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177303.757139:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177303.757141:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177303.757143:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177303.757145:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177303.757147:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177303.757150:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.757152:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177303.757154:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177303.757156:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177303.757158:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177303.757161:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177303.757163:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177303.757165:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177303.757168:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177303.757170:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.757173:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177303.757175:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177303.757178:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.757180:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177303.757184:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786177303.757187:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177303.757190:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177303.757192:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177303.757200:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177303.757202:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177303.757204:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177303.757210:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786177303.757214:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177303.757215:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1786177303.757220:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177303.757225:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177303.757230:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177303.757232:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177303.757236:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177303.757245:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786177303.757247:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177303.757283:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177303.757286:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177303.757291:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177303.757292:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11988) 02000000:00000001:0.0:1786177303.757296:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177303.757301:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177303.757316:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177303.757321:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235600640:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177303.757330:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177303.757335:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177303.757338:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177303.757341:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177303.757347:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177303.757350:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177303.757446:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177303.757455:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e210. 00000400:00000010:0.0:1786177303.757463:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686d48. 00000100:00000200:0.0:1786177303.757469:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235600640, portal 25 00000100:00000001:0.0:1786177303.757471:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177303.757473:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280821632 : -110371428729984 : ffff9b9e26327b80) 00000100:00000040:0.0:1786177303.757479:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177303.757487:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177303.757490:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177303.757498:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177303.757501:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235600640, offset 0 00000400:00000200:0.0:1786177303.757509:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177303.757517:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177303.757528:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177303.757533:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177303.757537:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177303.757541:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177303.757543:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177303.757546:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426759:426759:256:4294967295] 192.168.202.132@tcp LPNI seq info [426721:426721:8:4294967295] 00000400:00000200:0.0:1786177303.757551:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177303.757560:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177303.757565:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177303.757570:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177303.757577:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177303.757587:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177303.757746:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177303.757751:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177303.757754:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306654465 00000100:00000010:0.0:1786177303.757760:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5840. 00000100:00000001:0.0:1786177303.757766:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177303.757772:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306654465 00000100:00000010:0.0:1786177303.757776:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5ae0. 00000100:00000001:0.0:1786177303.757779:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177303.757782:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177303.759430:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177303.759435:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177303.759443:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177303.759452:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177303.759454:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177303.759457:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177303.759460:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177303.759462:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177303.759466:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600768:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177303.759469:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177303.759472:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177303.759473:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177303.759475:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177303.759481:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177303.759483:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177303.759488:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177303.759493:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e318. 00000400:00000010:0.0:1786177303.759499:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e316867f8. 00000100:00000200:0.0:1786177303.759503:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235600768, portal 10 00000100:00000001:0.0:1786177303.759506:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177303.759508:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280812672 : -110371428738944 : ffff9b9e26325880) 00000100:00000040:0.0:1786177303.759515:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177303.759522:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177303.759525:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177303.759527:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686550. 00000100:00000200:0.0:1786177303.759530:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235600768, offset 0 00000400:00000200:0.0:1786177303.759535:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177303.759540:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177303.759546:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177303.759550:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177303.759555:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177303.759559:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177303.759561:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177303.759565:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426760:426760:255:4294967295] 192.168.202.132@tcp LPNI seq info [426722:426722:7:4294967295] 00000400:00000200:0.0:1786177303.759571:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177303.759579:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177303.759584:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177303.759588:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c900. 00000800:00000200:0.0:1786177303.759594:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177303.759599:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177303.759603:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177303.759606:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177303.759610:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177303.759612:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306650630 00000100:00000001:0.0:1786177303.759615:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306654465 00000100:00000001:0.0:1786177303.759617:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306650630 00000100:00000001:0.0:1786177303.759618:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306654465 00000100:00000001:0.0:1786177303.759620:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177303.759622:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177303.759623:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177303.759627:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177303.759640:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177303.759641:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177303.759642:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177303.759644:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177303.759645:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177303.759647:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600896:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177303.759650:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177303.759652:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177303.759653:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177303.759654:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177303.759656:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177303.759658:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177303.759661:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177303.759663:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610ef78. 00000400:00000010:0.0:1786177303.759666:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686dd0. 00000100:00000200:0.0:1786177303.759669:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235600896, portal 10 00000100:00000001:0.0:1786177303.759670:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177303.759671:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280811776 : -110371428739840 : ffff9b9e26325500) 00000100:00000040:0.0:1786177303.759675:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177303.759680:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177303.759682:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177303.759684:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686330. 00000100:00000200:0.0:1786177303.759686:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235600896, offset 0 00000400:00000200:0.0:1786177303.759690:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177303.759693:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177303.759696:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177303.759698:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177303.759700:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177303.759702:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177303.759705:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177303.759707:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426761:426761:254:4294967295] 192.168.202.132@tcp LPNI seq info [426723:426723:6:4294967295] 00000400:00000200:0.0:1786177303.759710:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177303.759715:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177303.759719:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177303.759722:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c300. 00000800:00000200:0.0:1786177303.759725:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177303.759728:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177303.759731:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177303.759733:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177303.759736:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177303.759737:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306650630 00000100:00000001:0.0:1786177303.759740:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306654465 00000100:00000001:0.0:1786177303.759742:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306650630 00000100:00000001:0.0:1786177303.759743:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306654465 00000100:00000001:0.0:1786177303.759745:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177303.759747:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177303.759749:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177303.759753:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177303.759760:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177303.759761:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177303.759763:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177303.759765:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177303.759767:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177303.759770:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601024:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177303.759773:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177303.759775:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177303.759776:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177303.759777:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177303.759781:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177303.759783:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177303.759787:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177303.759790:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e738. 00000400:00000010:0.0:1786177303.759792:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686220. 00000100:00000200:0.0:1786177303.759795:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235601024, portal 4 00000100:00000001:0.0:1786177303.759796:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177303.759797:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280806400 : -110371428745216 : ffff9b9e26324000) 00000100:00000040:0.0:1786177303.759802:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177303.759807:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177303.759809:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177303.759811:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177303.759812:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235601024, offset 0 00000400:00000200:0.0:1786177303.759817:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177303.759819:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177303.759822:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177303.759824:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177303.759826:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177303.759828:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177303.759830:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177303.759832:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426762:426762:253:4294967295] 192.168.202.132@tcp LPNI seq info [426724:426724:5:4294967295] 00000400:00000200:0.0:1786177303.759835:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177303.759840:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177303.759844:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177303.759847:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c600. 00000800:00000200:0.0:1786177303.759850:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177303.759853:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177303.759856:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177303.759858:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177303.759860:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177303.759862:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306650630 00000100:00000001:0.0:1786177303.759863:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306654465 00000100:00000001:0.0:1786177303.759865:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306650630 00000100:00000001:0.0:1786177303.759867:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306654465 00000100:00000001:0.0:1786177303.759869:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177303.759871:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177303.759873:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177303.759876:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177303.759883:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177303.759886:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1786177303.759888:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177303.759890:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177303.759891:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177303.759895:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177303.759901:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177303.759902:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177303.759904:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177303.759906:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177303.759907:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177303.759909:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601152:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177303.759912:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177303.759914:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177303.759915:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177303.759917:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177303.759920:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177303.759922:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177303.759925:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177303.759927:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e8f0. 00000400:00000010:0.0:1786177303.759930:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e316865d8. 00000100:00000200:0.0:1786177303.759933:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235601152, portal 4 00000100:00000001:0.0:1786177303.759935:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177303.759936:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809088 : -110371428742528 : ffff9b9e26324a80) 00000100:00000040:0.0:1786177303.759941:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177303.759947:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177303.759949:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177303.759951:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177303.759953:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235601152, offset 0 00000400:00000200:0.0:1786177303.759957:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177303.759960:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177303.759963:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177303.759966:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177303.759968:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177303.759970:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177303.759972:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177303.759974:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426763:426763:252:4294967295] 192.168.202.132@tcp LPNI seq info [426725:426725:4:4294967295] 00000400:00000200:0.0:1786177303.759978:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177303.759983:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177303.759987:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177303.759990:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177303.759993:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177303.759997:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177303.760000:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177303.760002:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177303.760005:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177303.760007:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306650632 00000100:00000001:0.0:1786177303.760009:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306654465 00000100:00000001:0.0:1786177303.760011:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306650632 00000100:00000001:0.0:1786177303.760012:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306654465 00000100:00000001:0.0:1786177303.760014:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177303.760016:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177303.760019:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177303.760020:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177303.760023:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177303.760024:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1786177303.760025:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177303.760027:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177303.760029:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177303.760031:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177304.268556:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177304.268567:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177304.268569:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177304.268575:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177304.268580:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177304.268583:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177304.268586:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177304.268589:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177304.268593:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177304.652518:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177304.652518:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177304.652528:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177304.652530:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177304.652533:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177304.652537:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177304.652539:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177304.652539:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177304.652546:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177304.652546:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177304.652549:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177304.652551:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177304.652556:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177304.652558:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177304.652561:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177304.652671:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177304.652675:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177304.652679:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177305.292463:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177305.292474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177305.292477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177305.292485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177305.292490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177305.292495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177305.292498:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177305.292501:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177305.292504:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177305.676492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177305.676499:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177305.676506:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177305.676509:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177305.676513:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177305.676514:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177305.676570:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177305.676579:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177305.676582:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177305.676588:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177305.676593:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177305.676598:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177305.676601:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177305.676604:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177305.676608:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177305.676765:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177305.676770:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177305.676772:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177306.318150:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177306.318158:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177306.318160:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177306.318165:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177306.318168:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177306.318171:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177306.318174:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177306.318176:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177306.318179:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177306.700463:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177306.700470:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177306.700475:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177306.700477:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177306.700480:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177306.700482:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177306.700614:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177306.700619:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177306.700621:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177306.700625:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177306.700628:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177306.700632:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.700635:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177306.700637:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.700641:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177306.700911:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177306.700915:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177306.700919:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1786177306.830413:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:0.0:1786177306.830462:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1786177306.830472:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177306.830476:0:2380:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:0.0:1786177306.830481:0:2380:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177306.830485:0:2380:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1786177306.830490:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177306.831072:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:0.0:1786177306.831138:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177306.831145:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177306.831152:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:0.0:1786177306.831155:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:0.0:1786177306.831160:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177306.831168:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177306.831196:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177306.831200:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177306.831207:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177306.831209:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786177306.831218:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177306.831221:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c900. 00000400:00000200:0.0:1786177306.831224:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177306.831227:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177306.831229:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686550 00000400:00000010:0.0:1786177306.831231:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686550. 00000100:00000001:0.0:1786177306.831234:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177306.831238:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177306.831246:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177306.831249:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177306.831255:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177306.831257:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786177306.831262:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177306.831265:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c300. 00000400:00000200:0.0:1786177306.831268:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177306.831281:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177306.831284:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686330 00000400:00000010:0.0:1786177306.831286:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686330. 00000100:00000001:0.0:1786177306.831289:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177306.831293:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177306.831300:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177306.831303:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177306.831309:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177306.831310:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1786177306.831315:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177306.831327:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c600. 00000400:00000200:0.0:1786177306.831331:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177306.831334:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177306.831337:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177306.831339:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177306.831341:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177306.831345:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177306.831352:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177306.831354:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.831809:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177306.831814:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177306.831818:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177306.831826:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177303/real 1786177306] req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177306.831839:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177306.831841:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177306.831844:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e210. 00000400:00000200:1.0:1786177306.831848:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686d48 00000400:00000010:1.0:1786177306.831850:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686d48. 00000100:00000001:1.0:1786177306.831854:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177306.831858:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177306.831867:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.831875:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177306.831877:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.831879:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177306.831881:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177306.831885:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.831893:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177306.831897:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.831905:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177306.831908:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.831910:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177306.831911:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.831914:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177306.831916:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177306.831920:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177306.831924:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177306.831927:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177306.831931:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177306.831939:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235600640:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177306.831944:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177306.831948:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177303/real 1786177306] req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177306.831956:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177306.831957:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177306.831959:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e318. 00000400:00000200:1.0:1786177306.831961:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e316867f8 00000400:00000010:1.0:1786177306.831963:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e316867f8. 00000100:00000001:1.0:1786177306.831965:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177306.831968:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177306.831976:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.831982:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177306.831983:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.831985:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177306.831986:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177306.831989:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.831995:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177306.831998:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832005:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177306.832008:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832009:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177306.832010:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832011:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177306.832014:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177306.832016:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177306.832019:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177306.832021:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177306.832024:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177306.832031:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600768:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177306.832035:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177306.832039:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177303/real 1786177306] req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177306.832046:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177306.832047:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177306.832048:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610ef78. 00000400:00000200:1.0:1786177306.832050:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686dd0 00000400:00000010:1.0:1786177306.832052:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686dd0. 00000100:00000001:1.0:1786177306.832054:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177306.832058:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177306.832066:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832073:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177306.832074:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832076:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177306.832077:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177306.832080:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832087:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177306.832090:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832097:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177306.832099:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832101:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177306.832102:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832104:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177306.832106:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177306.832108:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177306.832110:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177306.832112:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177306.832116:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177306.832123:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235600896:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177306.832127:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177306.832131:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177303/real 1786177306] req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177306.832137:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177306.832138:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177306.832140:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e738. 00000400:00000200:1.0:1786177306.832142:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686220 00000400:00000010:1.0:1786177306.832143:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686220. 00000100:00000001:1.0:1786177306.832145:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177306.832148:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177306.832250:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832258:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177306.832259:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832261:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177306.832262:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177306.832266:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832282:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177306.832286:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 2 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832292:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177306.832294:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832296:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177306.832297:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832299:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177306.832301:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177306.832303:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177306.832307:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177306.832309:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177306.832312:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 2 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177306.832320:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601024:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177306.832325:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177306.832328:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177306.832332:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327b80 x1872930235600640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832338:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177306.832341:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177306.832346:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:1.0:1786177306.832350:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177306.832352:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177306.832354:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177306.832379:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177306.832383:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa000. 02000000:00000001:1.0:1786177306.832386:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177306.832388:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177306.832390:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327b80. 00000100:00000001:1.0:1786177306.832393:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177306.832394:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832396:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177306.832400:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325880 x1872930235600768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832407:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177306.832408:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177306.832410:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:1.0:1786177306.832412:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177306.832413:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177306.832414:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177306.832416:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177306.832429:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9400. 02000000:00000001:1.0:1786177306.832435:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177306.832436:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177306.832437:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325880. 00000100:00000001:1.0:1786177306.832439:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177306.832440:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832442:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177306.832447:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325500 x1872930235600896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832453:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177306.832455:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177306.832457:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:1.0:1786177306.832460:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177306.832461:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177306.832464:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177306.832468:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177306.832470:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9000. 00000100:00000001:0.0:1786177306.832471:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177306.832473:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1786177306.832475:0:2380:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1786177306.832476:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177306.832480:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325500. 00000100:00000001:1.0:1786177306.832482:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177306.832484:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832486:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000800:00000100:0.0:1786177306.832489:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:0.0:1786177306.832493:0:2380:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177306.832496:0:2380:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1786177306.832499:0:2380:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1786177306.832503:0:2380:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177306.832505:0:2380:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177306.832507:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1786177306.832512:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324000 x1872930235601024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177306.832516:0:2380:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177306.832527:0:2380:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177306.832530:0:2380:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177328 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177306.832537:0:2380:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177306.832538:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177306.832546:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177306.832548:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177306.832549:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:1.0:1786177306.832552:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177306.832554:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177306.832555:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177306.832557:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177306.832559:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa800. 02000000:00000001:1.0:1786177306.832561:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177306.832563:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177306.832565:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324000. 00000100:00000001:1.0:1786177306.832568:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177306.832569:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832572:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177306.832574:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177306.832575:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177306.832577:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177306.832583:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177303/real 1786177306] req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177306.832590:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177306.832592:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177306.832595:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e8f0. 00000400:00000200:1.0:1786177306.832598:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e316865d8 00000400:00000010:1.0:1786177306.832600:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e316865d8. 00000100:00000001:1.0:1786177306.832603:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177306.832606:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177306.832615:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832623:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177306.832624:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832626:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177306.832628:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177306.832632:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832638:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177306.832643:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832649:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177306.832651:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832653:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177306.832654:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832656:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177306.832657:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177306.832660:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177306.832663:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177306.832665:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177306.832669:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177306.832683:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601152:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177306.832687:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832689:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177306.832691:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324a80 x1872930235601152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177328 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177306.832697:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177306.832699:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177306.832700:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:1.0:1786177306.832702:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177306.832703:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177306.832704:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177306.832706:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177306.832708:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 02000000:00000001:1.0:1786177306.832710:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177306.832711:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177306.832713:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324a80. 00000100:00000001:1.0:1786177306.832715:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177306.832716:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832718:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177306.832724:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177306.832726:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177306.832727:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177306.832729:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177307.340730:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177307.340738:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177307.340740:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177307.340745:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177307.340748:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177307.340751:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177307.340754:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177307.340757:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177307.340760:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177307.660474:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306654465, at 4306654528 00000100:00000040:1.0:1786177307.660474:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306654465, at 4306654528 00000100:00000010:0.0:1786177307.660490:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5840. 00000100:00000010:1.0:1786177307.660492:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5ae0. 00000100:00000001:1.0:1786177307.724465:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177307.724473:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177307.724475:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177307.724480:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177307.724484:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177307.724487:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177307.724490:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177307.724492:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177307.724495:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177307.724587:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177307.724591:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177307.724596:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177307.724597:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177307.724601:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177307.724602:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177307.725038:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177307.725042:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177307.725046:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177308.364471:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177308.364479:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177308.364481:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177308.364487:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177308.364491:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177308.364504:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177308.364507:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177308.364509:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177308.364512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177308.748492:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177308.748501:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177308.748506:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177308.748507:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177308.748519:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177308.748520:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.748532:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177308.748537:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177308.748539:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177308.748543:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177308.748547:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177308.748550:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.748552:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177308.748555:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.748557:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177308.748585:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177308.748590:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177308.748593:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177308.876580:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177308.876590:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177308.876593:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177308.876596:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177308.876600:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177308.876607:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11983 00000400:00000200:3.0:1786177308.876615:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177308.876621:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177308.876624:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177308.876627:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.876631:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177308.876633:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.876636:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177308.876640:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177308.876642:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177308.876644:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177308.876647:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177308.876650:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177308.876653:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177308.876655:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.876657:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177308.876659:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.876661:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177308.876664:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177308.876667:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177308.876669:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177308.876671:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177308.876674:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177308.876677:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.876679:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177308.876681:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177308.876686:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786177308.876689:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177308.876693:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177308.876695:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.876697:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177308.876699:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177308.876703:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177308.876709:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786177308.876715:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177308.876723:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177308.876738:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177308.876743:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177308.876747:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.876796:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177308.876800:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177308.876803:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177308.876809:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.876817:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177308.876819:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177308.876822:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177308.876824:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177308.876827:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177308.876830:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235601280:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177308.876834:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177308.876838:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177308.876840:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177308.876842:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177308.876847:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177308.876849:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177308.876854:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177308.876857:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177308.876863:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177308.876870:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235601280, portal 25 00000100:00000001:1.0:1786177308.876872:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177308.876874:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809984 : -110371428741632 : ffff9b9e26324e00) 00000100:00000040:1.0:1786177308.876879:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.876887:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177308.876889:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177308.876892:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177308.876895:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235601280, offset 0 00000400:00000200:1.0:1786177308.876900:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177308.876907:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177308.876913:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177308.876918:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177308.876922:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177308.876926:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177308.876929:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177308.876933:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426764:426764:256:4294967295] 192.168.202.132@tcp LPNI seq info [426726:426726:8:4294967295] 00000400:00000200:1.0:1786177308.876937:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177308.876945:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177308.876951:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177308.876955:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177308.876961:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177308.876965:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177308.876970:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177308.876974:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177308.876978:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177308.876980:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177308.876984:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177308.876988:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177308.876993:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177308.876995:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177308.876999:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177308.877003:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177308.877027:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177308.877030:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177308.877033:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177308.877034:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177308.877036:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177308.877039:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177308.877041:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11983 00000400:00000200:3.0:1786177308.877045:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177308.877048:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177308.877050:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177308.877052:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877054:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177308.877057:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877059:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177308.877061:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177308.877063:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177308.877064:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177308.877066:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177308.877068:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177308.877070:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177308.877073:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877075:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177308.877077:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877079:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177308.877082:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177308.877085:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877087:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177308.877089:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177308.877094:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177308.877096:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877097:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177308.877101:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177308.877104:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877106:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177308.877109:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786177308.877111:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177308.877114:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177308.877115:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877117:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177308.877118:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177308.877120:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177308.877123:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786177308.877127:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177308.877132:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177308.877139:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177308.877141:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177308.877144:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.877280:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177308.877286:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.877300:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.877302:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177308.877310:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177308.877314:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177308.877317:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306659584 00000100:00000010:1.0:1786177308.877323:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c624e0. 00000100:00000001:1.0:1786177308.877330:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177308.877337:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306659584 00000100:00000010:1.0:1786177308.877340:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c620c0. 00000100:00000001:1.0:1786177308.877343:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177308.877346:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177308.877349:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.877351:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177308.877355:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177308/real 1786177308] req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177308.877385:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177308.877387:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177308.877389:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177308.877391:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177308.877393:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177308.877397:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177308.877401:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177308.877409:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.877416:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177308.877418:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.877420:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177308.877423:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177308.877427:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.877435:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177308.877439:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.877446:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177308.877448:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.877450:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177308.877451:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.877453:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177308.877456:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177308.877459:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177308.877463:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177308.877465:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177308.877469:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177308.877478:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235601280:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177308.877483:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.877485:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1786177308.877583:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177308.877587:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177308.877590:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177308.877591:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177308.877593:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177308.877595:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177308.877599:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11983 00000400:00000200:3.0:1786177308.877603:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177308.877606:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177308.877608:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177308.877611:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877613:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177308.877615:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877616:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177308.877618:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177308.877620:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177308.877621:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177308.877623:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177308.877624:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177308.877626:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177308.877628:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877629:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177308.877631:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877633:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177308.877635:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177308.877637:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877639:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177308.877640:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177308.877643:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177308.877645:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877647:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177308.877649:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177308.877651:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877652:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177308.877655:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786177308.877657:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177308.877660:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177308.877661:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877663:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177308.877664:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177308.877666:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177308.877669:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177308.877672:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177308.877677:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177308.877683:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177308.877685:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177308.877688:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177308.877695:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177308.877697:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177308.877700:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177308.877701:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177308.877703:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177308.877706:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177308.877708:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11983 00000400:00000200:3.0:1786177308.877711:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177308.877713:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177308.877715:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177308.877717:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877719:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177308.877721:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877722:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177308.877724:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177308.877725:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177308.877727:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177308.877728:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177308.877730:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177308.877732:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177308.877735:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877737:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177308.877738:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877740:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177308.877743:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177308.877745:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877748:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177308.877749:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177308.877751:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177308.877753:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877755:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177308.877756:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177308.877758:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877760:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177308.877764:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786177308.877766:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177308.877768:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177308.877769:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877771:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177308.877772:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177308.877773:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177308.877777:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786177308.877779:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177308.877783:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177308.877789:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177308.877791:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177308.877795:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177308.877802:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177308.877805:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177308.877808:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177308.877810:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177308.877811:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177308.877813:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177308.877815:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11983 00000400:00000200:3.0:1786177308.877818:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177308.877821:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177308.877822:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177308.877825:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877826:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177308.877828:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877830:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177308.877831:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177308.877832:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177308.877833:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177308.877835:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177308.877837:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177308.877839:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177308.877842:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877843:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177308.877845:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877847:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177308.877849:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177308.877851:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177308.877853:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177308.877854:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177308.877857:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177308.877858:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877860:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177308.877862:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177308.877864:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877866:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177308.877869:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786177308.877870:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177308.877872:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177308.877873:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177308.877875:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177308.877876:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177308.877877:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177308.877881:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786177308.877883:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177308.877887:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177308.877892:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177308.877894:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177308.877897:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177308.877903:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177308.877907:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11993) 00000100:00000040:1.0:1786177308.878169:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324e00 x1872930235601280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.878179:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177308.878182:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177308.878185:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177308.878188:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177308.878190:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177308.878192:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177308.878194:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177308.878196:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:1.0:1786177308.878200:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177308.878202:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177308.878204:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324e00. 00000100:00000001:1.0:1786177308.878207:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177308.878209:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.878211:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.878221:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177308.878223:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.878224:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177308.878226:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177308.878227:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177308.878232:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.878238:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177308.878239:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177308.878242:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177308.878244:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177308.878246:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177308.878249:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601408:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177308.878252:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177308.878265:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177308.878267:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177308.878269:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177308.878272:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177308.878274:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177308.878278:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177308.878281:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177308.878286:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177308.878290:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235601408, portal 10 00000100:00000001:1.0:1786177308.878292:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177308.878293:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280817152 : -110371428734464 : ffff9b9e26326a00) 00000100:00000040:1.0:1786177308.878299:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.878305:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177308.878307:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177308.878311:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177308.878313:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235601408, offset 0 00000400:00000200:1.0:1786177308.878318:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177308.878323:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177308.878328:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177308.878332:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177308.878335:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177308.878338:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177308.878340:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177308.878343:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426765:426765:256:4294967295] 192.168.202.132@tcp LPNI seq info [426727:426727:8:4294967295] 00000400:00000200:1.0:1786177308.878347:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177308.878353:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177308.878383:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177308.878387:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177308.878392:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177308.878395:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177308.878399:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177308.878402:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177308.878405:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177308.878408:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177308.878411:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177308.878415:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177308.878419:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177308.878421:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177308.878423:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177308.878427:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.878435:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177308.878439:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.878444:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.878445:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177308.878450:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177308.878454:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177308.878457:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177308.878459:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306655749 00000100:00000001:1.0:1786177308.878462:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306659584 00000100:00000001:1.0:1786177308.878464:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306655749 00000100:00000001:1.0:1786177308.878465:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306659584 00000100:00000001:1.0:1786177308.878467:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177308.878469:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.878471:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177308.878893:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177308/real 1786177308] req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177308.878905:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177308.878906:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177308.878909:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177308.878912:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177308.878914:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177308.878916:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177308.878920:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177308.878927:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.878934:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177308.878935:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.878938:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177308.878940:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177308.878943:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.878949:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177308.878953:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.878960:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177308.878962:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.878964:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177308.878965:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.878967:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177308.878970:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177308.878973:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177308.878977:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177308.878979:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177308.878983:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177308.878990:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601408:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177308.878995:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177308.878998:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879004:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177308.879005:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177308.879007:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177308.879009:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177308.879011:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177308.879014:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601536:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177308.879018:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177308.879022:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177308.879024:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177308.879026:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177308.879030:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177308.879032:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177308.879036:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177308.879039:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177308.879042:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177308.879046:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235601536, portal 10 00000100:00000001:1.0:1786177308.879048:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177308.879049:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280808192 : -110371428743424 : ffff9b9e26324700) 00000100:00000040:1.0:1786177308.879055:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879061:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177308.879063:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177308.879066:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177308.879069:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235601536, offset 0 00000400:00000200:1.0:1786177308.879073:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177308.879077:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177308.879081:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177308.879085:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177308.879087:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177308.879090:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177308.879092:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177308.879096:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426766:426766:256:4294967295] 192.168.202.132@tcp LPNI seq info [426728:426728:8:4294967295] 00000400:00000200:1.0:1786177308.879099:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177308.879105:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177308.879110:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177308.879113:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177308.879117:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177308.879120:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177308.879124:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177308.879127:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177308.879131:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177308.879133:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177308.879136:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177308.879139:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177308.879142:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177308.879144:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177308.879147:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177308.879150:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879158:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177308.879160:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879166:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.879168:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177308.879172:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177308.879175:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177308.879178:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177308.879180:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306655749 00000100:00000001:1.0:1786177308.879182:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306659584 00000100:00000001:1.0:1786177308.879184:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306655749 00000100:00000001:1.0:1786177308.879186:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306659584 00000100:00000001:1.0:1786177308.879188:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177308.879190:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.879192:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177308.879196:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177308/real 1786177308] req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177308.879203:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177308.879204:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177308.879205:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177308.879207:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177308.879208:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177308.879210:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177308.879214:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177308.879222:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879227:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177308.879229:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.879231:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177308.879232:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177308.879236:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879242:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177308.879246:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879252:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177308.879266:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.879267:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177308.879269:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.879271:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177308.879273:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177308.879275:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177308.879279:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177308.879281:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177308.879284:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177308.879292:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601536:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177308.879297:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177308.879299:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879305:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177308.879307:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177308.879309:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177308.879311:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177308.879313:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177308.879316:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601664:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177308.879318:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177308.879321:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177308.879322:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177308.879324:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177308.879327:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177308.879329:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177308.879333:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177308.879335:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177308.879338:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177308.879341:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235601664, portal 4 00000100:00000001:1.0:1786177308.879342:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177308.879343:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809088 : -110371428742528 : ffff9b9e26324a80) 00000100:00000040:1.0:1786177308.879348:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879354:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177308.879700:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177308.879704:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177308.879707:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235601664, offset 0 00000400:00000200:1.0:1786177308.879711:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177308.879716:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177308.879719:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177308.879722:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177308.879725:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177308.879727:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177308.879729:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177308.879733:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426767:426767:256:4294967295] 192.168.202.132@tcp LPNI seq info [426729:426729:8:4294967295] 00000400:00000200:1.0:1786177308.879737:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177308.879744:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177308.879748:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177308.879751:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177308.879755:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177308.879757:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177308.879762:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177308.879764:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177308.879767:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177308.879769:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177308.879772:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177308.879775:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177308.879778:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177308.879780:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177308.879782:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177308.879786:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879794:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177308.879797:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879803:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.879805:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177308.879808:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177308.879811:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177308.879813:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177308.879815:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306655750 00000100:00000001:1.0:1786177308.879818:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306659584 00000100:00000001:1.0:1786177308.879820:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306655750 00000100:00000001:1.0:1786177308.879822:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306659584 00000100:00000001:1.0:1786177308.879824:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177308.879825:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.879827:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177308.879830:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177308/real 1786177308] req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177308.879839:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177308.879840:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177308.879841:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177308.879843:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177308.879845:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177308.879847:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177308.879850:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177308.879858:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879864:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177308.879865:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.879867:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177308.879868:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177308.879872:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879878:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177308.879881:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879887:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177308.879889:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.879891:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177308.879892:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.879894:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177308.879896:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177308.879898:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177308.879901:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177308.879903:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177308.879907:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177308.879915:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601664:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177308.879918:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177308.879921:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879928:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177308.879930:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177308.879932:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177308.879934:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177308.879936:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177308.879939:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601792:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177308.879941:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177308.879943:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177308.879945:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177308.879947:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177308.879950:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177308.879952:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177308.879955:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177308.879957:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177308.879959:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177308.879962:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235601792, portal 4 00000100:00000001:1.0:1786177308.879964:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177308.879965:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280806400 : -110371428745216 : ffff9b9e26324000) 00000100:00000040:1.0:1786177308.879971:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.879976:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177308.879978:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177308.879981:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177308.879983:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235601792, offset 0 00000400:00000200:1.0:1786177308.879987:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177308.879991:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177308.879994:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177308.879996:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177308.879998:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177308.880000:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177308.880003:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177308.880006:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426768:426768:256:4294967295] 192.168.202.132@tcp LPNI seq info [426730:426730:8:4294967295] 00000400:00000200:1.0:1786177308.880009:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177308.880014:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177308.880019:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177308.880022:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177308.880026:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177308.880028:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1786177308.880032:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177308.880034:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1786177308.880038:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177308.880040:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177308.880042:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177308.880045:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177308.880048:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177308.880049:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177308.880052:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177308.880055:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.880062:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177308.880065:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177333 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.880072:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.880073:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177308.880076:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177308.880079:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177308.880082:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177308.880085:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306655752 00000100:00000001:1.0:1786177308.880087:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306659584 00000100:00000001:1.0:1786177308.880089:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306655752 00000100:00000001:1.0:1786177308.880091:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306659584 00000100:00000001:1.0:1786177308.880093:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177308.880095:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.880097:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177308.880101:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177308/real 1786177308] req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177308.880108:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177308.880109:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177308.880111:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177308.880113:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177308.880115:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177308.880117:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177308.880119:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177308.880127:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.880134:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177308.880136:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.880137:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177308.880139:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177308.880142:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.880148:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177308.880152:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.880159:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177308.880161:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.880163:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177308.880164:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.880165:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177308.880167:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177308.880169:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177308.880173:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177308.880175:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177308.880178:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177308.880186:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235601792:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177308.880189:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.880192:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177308.880195:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326a00 x1872930235601408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.880202:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177308.880204:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177308.880206:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:1.0:1786177308.880208:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177308.880210:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177308.880212:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177308.880214:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177308.880217:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:1.0:1786177308.880220:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177308.880221:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177308.880223:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326a00. 00000100:00000001:1.0:1786177308.880225:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177308.880226:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.880228:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177308.880232:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324700 x1872930235601536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.880244:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177308.880245:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177308.880247:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177308.880249:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177308.880250:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177308.880251:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177308.880515:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177308.880519:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:1.0:1786177308.880522:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177308.880524:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177308.880525:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324700. 00000100:00000001:1.0:1786177308.880528:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177308.880529:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.880532:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177308.880536:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324a80 x1872930235601664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.880543:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177308.880544:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177308.880546:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177308.880548:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177308.880549:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177308.880551:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177308.880552:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177308.880553:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:1.0:1786177308.880555:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177308.880556:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177308.880557:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324a80. 00000100:00000001:1.0:1786177308.880559:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177308.880560:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.880562:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177308.880565:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324000 x1872930235601792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177333 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177308.880570:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177308.880571:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177308.880573:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:1.0:1786177308.880575:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177308.880576:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177308.880578:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177308.880580:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177308.880581:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8400. 02000000:00000001:1.0:1786177308.880583:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177308.880584:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177308.880585:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324000. 00000100:00000001:1.0:1786177308.880586:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177308.880588:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.880590:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177308.880597:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177308.880599:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177308.880601:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177308.880603:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177309.388536:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177309.388544:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177309.388547:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177309.388553:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177309.388557:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177309.388561:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177309.388563:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177309.388566:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177309.388570:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177309.772469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177309.772476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177309.772478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177309.772483:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177309.772487:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177309.772491:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177309.772493:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177309.772496:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177309.772498:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177309.772632:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177309.772636:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177309.772640:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177309.772642:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177309.772645:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177309.772646:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177309.772783:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177309.772786:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177309.772789:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177310.412576:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177310.412584:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177310.412587:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177310.412592:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177310.412596:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177310.412600:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177310.412603:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177310.412606:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177310.412609:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177310.796496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177310.796504:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177310.796511:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177310.796514:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177310.796516:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177310.796520:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177310.796523:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177310.796525:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177310.796527:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177310.796530:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177310.796533:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177310.796535:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177310.796616:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177310.796620:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177310.796623:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177310.796752:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177310.796756:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177310.796758:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177310.796762:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177310.796765:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177310.796768:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177310.796771:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177310.796773:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177310.796776:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177311.436494:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177311.436502:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177311.436505:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177311.436512:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177311.436517:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177311.436521:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177311.436523:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177311.436526:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177311.436530:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177311.820523:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177311.820533:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177311.820541:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177311.820542:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177311.820546:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177311.820547:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177311.820788:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177311.820797:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177311.820799:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177311.820804:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177311.820807:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177311.820811:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177311.820813:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177311.820815:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177311.820817:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177311.823406:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177311.823414:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177311.823420:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177312.460496:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177312.460505:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177312.460508:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177312.460514:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177312.460518:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177312.460522:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177312.460525:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177312.460527:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177312.460530:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177312.780512:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306659584, at 4306659648 00000100:00000010:1.0:1786177312.780535:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c620c0. 00000100:00000040:0.0:1786177312.781111:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306659584, at 4306659648 00000100:00000010:0.0:1786177312.781120:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c624e0. 00010000:00000001:0.0:1786177312.844517:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177312.844527:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177312.844534:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177312.844536:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177312.844540:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177312.844541:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177312.844552:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177312.844556:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177312.844558:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177312.844562:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177312.844566:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177312.844569:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177312.844572:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177312.844574:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177312.844577:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177312.844628:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177312.844700:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177312.844704:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786177313.420540:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786177313.420551:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786177313.420557:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:1.0:1786177313.420776:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1786177313.420784:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1786177313.420797:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1786177313.420801:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1786177313.420804:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177313.484450:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177313.484459:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177313.484461:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177313.484468:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177313.484472:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177313.484475:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177313.484478:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177313.484481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177313.484483:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177313.868489:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177313.868502:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177313.868505:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177313.868509:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177313.868515:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177313.868518:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177313.868521:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177313.868524:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177313.868530:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177313.868533:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177313.868538:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177313.868540:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177313.868695:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177313.868697:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177313.868700:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177313.868701:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177313.868703:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177313.868705:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177313.868764:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177313.868769:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177313.868773:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177313.870410:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177313.870417:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177313.870418:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177313.870424:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177313.870428:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177313.870431:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177313.870434:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177313.870437:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177313.870440:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177313.996511:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177313.996523:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177313.996525:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177313.996529:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177313.996534:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177313.996541:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11988 00000400:00000200:3.0:1786177313.996551:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177313.996560:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177313.996563:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177313.996566:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.996572:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177313.996575:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.996577:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177313.996582:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177313.996584:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177313.996587:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177313.996590:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177313.996592:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177313.996595:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177313.996599:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.996601:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177313.996604:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.996606:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177313.996610:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177313.996613:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177313.996616:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177313.996619:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177313.996623:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177313.996625:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177313.996628:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177313.996631:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177313.996640:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786177313.996644:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177313.996648:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177313.996650:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177313.996653:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177313.996656:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177313.996660:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177313.996681:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786177313.996688:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177313.996697:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177313.996708:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177313.996713:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177313.996717:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177313.996754:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177313.996757:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177313.996761:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177313.996762:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177313.996764:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177313.996767:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177313.996769:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11988 00000400:00000200:3.0:1786177313.996773:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177313.996776:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177313.996779:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177313.996782:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.996785:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177313.996788:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.996790:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177313.996794:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177313.996796:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1786177313.996796:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177313.996798:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177313.996803:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1786177313.996803:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00000001:1.0:1786177313.996807:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1786177313.996808:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177313.996813:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177313.996818:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177313.996819:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.996823:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177313.996828:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.996832:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177313.996837:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:1.0:1786177313.996839:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1786177313.996842:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177313.996843:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.996849:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1786177313.996849:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.996852:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177313.996853:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177313.996855:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177313.996859:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235601920:192.168.202.132@tcp:250:kworker.0 02000000:00000001:3.0:1786177313.996860:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177313.996864:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1786177313.996866:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.996869:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1786177313.996869:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1786177313.996873:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1786177313.996875:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1786177313.996876:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1786177313.996881:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177313.996883:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 00000100:00000001:3.0:1786177313.996884:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.996888:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177313.996891:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786177313.996895:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000200:1.0:1786177313.996898:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:3.0:1786177313.996899:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177313.996902:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177313.996904:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000100:00000001:3.0:1786177313.996905:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177313.996909:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177313.996911:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177313.996913:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 02000000:00000010:3.0:1786177313.996916:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786177313.996921:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177313.996923:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235601920, portal 25 00000100:00000001:1.0:1786177313.996926:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177313.996928:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280811776 : -110371428739840 : ffff9b9e26325500) 00000100:00100000:3.0:1786177313.996930:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1786177313.996935:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177313.996951:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177313.996952:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1786177313.996956:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177313.996957:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177313.996962:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177313.996965:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235601920, offset 0 00000100:00000040:3.0:1786177313.996966:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177313.996975:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177313.996984:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177313.996985:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:3.0:1786177313.996990:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1786177313.996997:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00080000:3.0:1786177313.996997:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177313.997001:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177313.997004:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1786177313.997005:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1786177313.997009:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177313.997013:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00080000:3.0:1786177313.997017:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11988 00000400:00000200:1.0:1786177313.997022:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1786177313.997026:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177313.997029:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:3.0:1786177313.997033:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177313.997037:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177313.997038:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426769:426769:256:4294967295] 192.168.202.132@tcp LPNI seq info [426731:426731:8:4294967295] 00000100:00000040:3.0:1786177313.997042:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.997048:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177313.997049:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177313.997053:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.997057:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177313.997061:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177313.997064:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177313.997068:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1786177313.997069:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000020:00000040:3.0:1786177313.997072:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177313.997075:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000200:1.0:1786177313.997079:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000020:00000001:3.0:1786177313.997080:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000010:1.0:1786177313.997084:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000040:3.0:1786177313.997086:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.997090:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177313.997094:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:1.0:1786177313.997096:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177313.997098:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:1.0:1786177313.997100:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000020:00000040:3.0:1786177313.997102:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000800:00000200:1.1:1786177313.997107:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786177313.997110:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:1.1:1786177313.997111:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1786177313.997118:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1786177313.997119:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177313.997122:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 02000000:00000001:3.0:1786177313.997122:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177313.997128:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1786177313.997130:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:3.0:1786177313.997133:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177313.997138:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1786177313.997139:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000008:00100000:3.0:1786177313.997144:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1786177313.997145:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177313.997149:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000008:00000001:3.0:1786177313.997150:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177313.997154:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.997155:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:3.0:1786177313.997158:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000100:00000200:1.0:1786177313.997162:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177313.997163:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177313.997169:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177313.997171:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177313.997176:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177313.997179:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177313.997181:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1786177313.997182:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177313.997188:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1786177313.997189:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786177313.997197:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.997202:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177313.997220:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177313.997222:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1786177313.997231:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177313.997232:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177313.997234:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177313.997239:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177313.997241:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.997244:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306664704 00000100:00000010:1.0:1786177313.997253:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c620c0. 00000100:00000001:3.0:1786177313.997254:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177313.997259:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177313.997263:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177313.997264:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1786177313.997265:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177313.997269:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177313.997272:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1786177313.997274:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306664704 00000100:00000010:1.0:1786177313.997279:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c622a0. 00000100:00000001:1.0:1786177313.997282:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177313.997287:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177313.997290:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.997292:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177313.997298:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177313/real 1786177313] req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177313.997307:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177313.997309:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177313.997311:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177313.997314:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177313.997316:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177313.997318:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177313.997322:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177313.997331:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.997336:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177313.997338:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.997340:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177313.997342:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177313.997352:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.997408:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177313.997414:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.997421:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177313.997423:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.997425:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177313.997427:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.997429:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177313.997431:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177313.997434:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177313.997437:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177313.997440:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177313.997444:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177313.997453:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235601920:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177313.997458:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.997460:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177313.997464:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325500 x1872930235601920/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.997476:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177313.997478:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177313.997481:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177313.997484:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177313.997486:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177313.997487:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177313.997489:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177313.997492:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:1.0:1786177313.997495:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177313.997496:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177313.997497:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325500. 00000100:00000001:1.0:1786177313.997499:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177313.997499:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.997501:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.997506:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177313.997507:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.997509:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177313.997510:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177313.997511:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177313.997514:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.997519:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177313.997520:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.997522:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177313.997524:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177313.997526:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177313.997529:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602048:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177313.997532:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177313.997534:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177313.997536:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.997538:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177313.997542:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177313.997544:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177313.997548:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177313.997552:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177313.997556:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177313.997559:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235602048, portal 10 00000100:00000001:1.0:1786177313.997561:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177313.997562:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280812672 : -110371428738944 : ffff9b9e26325880) 00000100:00000040:1.0:1786177313.997569:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.997575:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177313.997577:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177313.997579:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177313.997582:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235602048, offset 0 00000400:00000200:1.0:1786177313.997585:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177313.997589:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177313.997591:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177313.997594:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177313.997595:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177313.997597:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177313.997599:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177313.997601:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426770:426770:256:4294967295] 192.168.202.132@tcp LPNI seq info [426732:426732:8:4294967295] 00000400:00000200:1.0:1786177313.997604:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177313.997608:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177313.997611:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177313.997615:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177313.997620:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177313.997623:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786177313.997629:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177313.997631:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1786177313.997636:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177313.997638:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177313.997641:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177313.997647:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177313.997652:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177313.997655:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177313.997658:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177313.997662:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.997671:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177313.997675:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.997680:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.997682:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177313.997686:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177313.997690:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177313.997693:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177313.997696:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306660868 00000100:00000001:1.0:1786177313.997699:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306664704 00000100:00000001:1.0:1786177313.997703:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306660868 00000100:00000001:1.0:1786177313.997706:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306664704 00000100:00000001:1.0:1786177313.997708:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177313.997710:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.997712:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1786177313.997718:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11988 00000100:00100000:1.0:1786177313.997720:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177313/real 1786177313] req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1786177313.997729:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000001:1.0:1786177313.997737:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:3.0:1786177313.997740:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000001:1.0:1786177313.997742:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1786177313.997744:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1786177313.997746:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000100:00000040:3.0:1786177313.997750:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177313.997752:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177313.997756:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177313.997758:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177313.997762:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177313.997771:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.997784:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177313.997785:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.997788:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177313.997790:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177313.997794:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1786177313.997795:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177313.997800:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.997806:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177313.997810:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:1.0:1786177313.997812:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1786177313.997813:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177313.997817:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177313.997820:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000040:1.0:1786177313.997821:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177313.997824:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177313.997829:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177313.997836:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177313.997839:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177313.997841:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1786177313.997845:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177313.997846:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177313.997849:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1786177313.997851:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177313.997853:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1786177313.997856:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:1.0:1786177313.997857:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177313.997862:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1786177313.997863:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00080000:1.0:1786177313.997868:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1786177313.997869:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177313.997874:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:1.0:1786177313.997875:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 02000000:00000001:3.0:1786177313.997879:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177313.997880:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1786177313.997885:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177313.997889:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177313.997889:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177313.997893:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177313.997898:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177313.997902:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177313.997909:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000100:00100000:1.0:1786177313.997910:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602048:192.168.202.132@tcp:38:kworker.0 00000020:00000040:3.0:1786177313.997913:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:1.0:1786177313.997917:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1786177313.997919:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177313.997922:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177313.997923:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177313.997927:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177313.997931:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1786177313.997932:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177313.997935:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177313.997936:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.997938:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177313.997941:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177313.997944:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177313.997945:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00100000:1.0:1786177313.997946:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602176:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177313.997951:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177313.997954:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1786177313.997954:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.997958:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177313.997963:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177313.997972:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177313.997975:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.997980:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1786177313.997983:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177313.997986:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177313.997991:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177313.998000:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177313.998002:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177313.998004:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000100:00080000:3.0:1786177313.998008:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1786177313.998010:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00080000:3.0:1786177313.998016:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000200:1.0:1786177313.998017:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235602176, portal 10 00000100:00000001:3.0:1786177313.998019:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1786177313.998022:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1786177313.998023:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177313.998026:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280821632 : -110371428729984 : ffff9b9e26327b80) 00000100:00000001:3.0:1786177313.998028:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177313.998033:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11988 00000100:00000040:1.0:1786177313.998037:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1786177313.998041:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177313.998049:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1786177313.998049:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000040:1.0:1786177313.998054:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177313.998055:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1786177313.998057:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177313.998060:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235602176, offset 0 00000100:00000040:3.0:1786177313.998061:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177313.998067:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177313.998068:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177313.998073:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177313.998074:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177313.998079:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177313.998080:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000001:3.0:1786177313.998084:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1786177313.998087:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:3.0:1786177313.998087:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000400:00000200:1.0:1786177313.998091:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000001:3.0:1786177313.998092:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1786177313.998097:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1786177313.998098:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000400:00000200:1.0:1786177313.998103:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000001:3.0:1786177313.998104:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177313.998109:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000200:1.0:1786177313.998111:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426771:426771:256:4294967295] 192.168.202.132@tcp LPNI seq info [426733:426733:8:4294967295] 00000100:00000040:3.0:1786177313.998118:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.998121:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177313.998123:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177313.998125:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177313.998128:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000400:00000200:1.0:1786177313.998135:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:3.0:1786177313.998136:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177313.998143:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177313.998146:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177313.998151:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177313.998157:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177313.998162:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1786177313.998163:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000008:00100000:3.0:1786177313.998166:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177313.998172:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177313.998176:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177313.998177:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1786177313.998182:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000010:3.0:1786177313.998184:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000800:00000200:1.0:1786177313.998188:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000040:3.0:1786177313.998191:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000800:00000200:1.0:1786177313.998192:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 02000000:00000001:3.0:1786177313.998197:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177313.998202:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786177313.998202:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177313.998217:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1786177313.998219:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:1.0:1786177313.998224:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 02000000:00000001:3.0:1786177313.998224:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000010:1.0:1786177313.998226:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 02000000:00000001:3.0:1786177313.998229:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177313.998230:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177313.998234:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177313.998237:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 02000000:00000010:3.0:1786177313.998237:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000400:00000010:1.0:1786177313.998238:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177313.998243:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786177313.998245:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177313.998247:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1786177313.998254:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177313.998256:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177313.998261:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177313.998276:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177313.998278:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177313.998280:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.998282:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177313.998287:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177313.998288:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177313.998292:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177313.998300:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177313.998305:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306660868 00000100:00000001:3.0:1786177313.998307:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.998309:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306664704 00000100:00000040:3.0:1786177313.998313:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (11998) 00000100:00000001:1.0:1786177313.998314:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306660868 00000100:00000001:1.0:1786177313.998319:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306664704 00000100:00000001:1.0:1786177313.998322:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177313.998324:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.998326:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177313.998333:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177313/real 1786177313] req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177313.998342:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177313.998344:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177313.998346:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177313.998349:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177313.998351:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177313.998354:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177313.998381:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177313.998390:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998395:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177313.998396:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998397:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177313.998398:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177313.998401:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998405:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177313.998408:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998412:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177313.998414:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998415:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177313.998416:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998417:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177313.998419:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177313.998421:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177313.998423:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177313.998424:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177313.998428:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177313.998433:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602176:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177313.998435:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998436:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177313.998438:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325880 x1872930235602048/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998441:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177313.998442:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177313.998444:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177313.998446:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177313.998447:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177313.998448:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177313.998449:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177313.998451:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8c00. 02000000:00000001:1.0:1786177313.998452:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177313.998455:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177313.998456:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325880. 00000100:00000001:1.0:1786177313.998459:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177313.998460:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998462:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177313.998464:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327b80 x1872930235602176/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998468:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177313.998469:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177313.998471:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177313.998472:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177313.998473:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177313.998474:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177313.998475:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177313.998476:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab800. 02000000:00000001:1.0:1786177313.998477:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177313.998479:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177313.998480:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327b80. 00000100:00000001:1.0:1786177313.998484:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177313.998486:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998488:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998493:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177313.998495:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.998498:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177313.998499:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177313.998500:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177313.998504:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998511:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177313.998513:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.998515:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177313.998517:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177313.998519:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177313.998522:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602304:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177313.998524:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177313.998526:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177313.998528:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.998529:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177313.998532:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177313.998534:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177313.998538:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177313.998540:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177313.998543:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177313.998546:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235602304, portal 4 00000100:00000001:1.0:1786177313.998548:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177313.998549:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280815360 : -110371428736256 : ffff9b9e26326300) 00000100:00000040:1.0:1786177313.998555:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998561:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177313.998564:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177313.998566:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177313.998569:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235602304, offset 0 00000400:00000200:1.0:1786177313.998572:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177313.998575:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177313.998577:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177313.998579:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177313.998581:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177313.998582:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177313.998585:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177313.998588:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426772:426772:256:4294967295] 192.168.202.132@tcp LPNI seq info [426734:426734:8:4294967295] 00000400:00000200:1.0:1786177313.998592:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177313.998597:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177313.998604:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177313.998608:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177313.998613:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177313.998616:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786177313.998621:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177313.998624:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1786177313.998628:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177313.998630:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177313.998632:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177313.998635:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177313.998638:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177313.998640:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177313.998642:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177313.998646:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998654:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177313.998656:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998663:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.998664:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177313.998667:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177313.998670:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177313.998673:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177313.998675:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306660869 00000100:00000001:1.0:1786177313.998677:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306664704 00000100:00000001:1.0:1786177313.998680:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306660869 00000100:00000001:1.0:1786177313.998682:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306664704 00000100:00000001:1.0:1786177313.998683:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177313.998685:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.998687:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177313.998691:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177313/real 1786177313] req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177313.998699:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177313.998700:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177313.998702:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177313.998715:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177313.998717:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177313.998719:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177313.998723:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177313.998731:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998738:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177313.998739:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998741:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177313.998742:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177313.998745:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998749:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177313.998751:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998756:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177313.998757:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998758:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177313.998759:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998760:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177313.998761:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177313.998762:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177313.998764:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177313.998765:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177313.998767:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177313.998771:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602304:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177313.998774:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177313.998776:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998779:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177313.998780:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.998782:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177313.998783:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177313.998784:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177313.998785:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602432:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177313.998787:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177313.998788:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177313.998789:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177313.998790:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177313.998792:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177313.998794:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177313.998796:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177313.998798:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177313.998800:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177313.998802:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235602432, portal 4 00000100:00000001:1.0:1786177313.998803:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177313.998804:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280813568 : -110371428738048 : ffff9b9e26325c00) 00000100:00000040:1.0:1786177313.998808:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998812:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177313.998813:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177313.998815:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177313.998816:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235602432, offset 0 00000400:00000200:1.0:1786177313.998818:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177313.998820:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177313.998822:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177313.998824:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177313.998826:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177313.998827:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177313.998829:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177313.998831:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426773:426773:256:4294967295] 192.168.202.132@tcp LPNI seq info [426735:426735:8:4294967295] 00000400:00000200:1.0:1786177313.998833:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177313.998837:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177313.998839:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177313.998841:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177313.998843:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177313.998845:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1786177313.998848:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177313.998849:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1786177313.998851:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177313.998853:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177313.998855:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177313.998857:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177313.998859:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177313.998860:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177313.998862:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177313.998863:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998868:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177313.998870:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177338 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998874:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.998876:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177313.998878:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177313.998881:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177313.998884:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177313.998886:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306660871 00000100:00000001:1.0:1786177313.998888:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306664704 00000100:00000001:1.0:1786177313.998890:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306660871 00000100:00000001:1.0:1786177313.998892:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306664704 00000100:00000001:1.0:1786177313.998894:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177313.998895:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.998897:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177313.998901:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177313/real 1786177313] req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177313.998908:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177313.998910:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177313.998911:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177313.998913:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177313.998915:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177313.998917:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177313.998920:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177313.998926:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998934:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177313.998935:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998936:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177313.998937:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177313.998939:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998942:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177313.998944:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998949:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177313.998950:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998951:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177313.998951:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998952:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177313.998953:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177313.998954:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177313.998956:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177313.998957:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177313.998959:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177313.998963:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602432:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177313.998965:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998966:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177313.998968:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326300 x1872930235602304/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998972:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177313.998973:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177313.998975:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177313.998976:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177313.998977:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177313.998978:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177313.998979:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177313.998980:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa400. 02000000:00000001:1.0:1786177313.998982:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177313.998982:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177313.998983:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326300. 00000100:00000001:1.0:1786177313.998985:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177313.998986:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.998987:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177313.998988:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325c00 x1872930235602432/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177338 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177313.998992:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177313.998992:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177313.998993:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177313.998994:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177313.998995:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177313.998995:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177313.998996:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177313.998997:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:1.0:1786177313.998998:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177313.998998:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177313.998999:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325c00. 00000100:00000001:1.0:1786177313.999000:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177313.999001:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.999002:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177313.999004:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177313.999005:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177313.999006:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177313.999007:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177314.508487:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177314.508495:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177314.508498:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177314.508504:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177314.508507:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177314.508511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177314.508514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177314.508516:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177314.508519:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177314.892476:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177314.892484:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177314.892486:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177314.892491:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177314.892495:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177314.892498:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177314.892500:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177314.892503:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177314.892506:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177314.892838:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177314.892842:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177314.892846:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177314.892847:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177314.892851:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177314.892852:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177314.893062:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177314.893065:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177314.893067:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1786177315.532559:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177315.532567:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177315.532569:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177315.532574:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177315.532578:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177315.532581:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177315.532584:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177315.532587:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177315.532590:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177315.916633:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177315.916641:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177315.916647:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177315.916648:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177315.916652:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177315.916653:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177315.916762:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177315.916766:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177315.916768:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177315.916847:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177315.916851:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177315.916853:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177315.916857:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177315.916860:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177315.916863:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177315.916865:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177315.916867:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177315.916869:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177316.557011:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177316.557018:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177316.557019:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177316.557023:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177316.557025:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177316.557028:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177316.557030:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177316.557031:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177316.557033:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177316.940467:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177316.940474:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177316.940476:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177316.940481:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1786177316.940481:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177316.940485:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177316.940486:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177316.940488:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177316.940491:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177316.940493:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177316.940494:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177316.940496:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177316.940498:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177316.940502:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177316.940504:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177316.940521:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177316.940525:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177316.940529:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177317.580467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177317.580476:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177317.580479:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177317.580485:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177317.580490:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177317.580493:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177317.580495:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177317.580497:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177317.580500:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177317.900445:0:206592:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306664704, at 4306664768 00000100:00000040:1.0:1786177317.900445:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306664704, at 4306664768 00000100:00000010:0.0:1786177317.900461:0:206592:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c620c0. 00000100:00000010:1.0:1786177317.900461:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c622a0. 00010000:00000001:0.0:1786177317.964423:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177317.964429:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177317.964433:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177317.964434:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177317.964437:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177317.964437:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177317.964523:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177317.964527:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177317.964529:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177317.965381:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177317.965386:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177317.965388:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177317.965392:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177317.965396:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177317.965400:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177317.965403:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177317.965406:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177317.965409:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177318.604490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177318.604498:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177318.604500:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177318.604506:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177318.604510:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177318.604513:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177318.604516:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177318.604518:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177318.604522:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177318.988461:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177318.988467:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177318.988469:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177318.988473:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177318.988473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177318.988480:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177318.988483:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177318.988484:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177318.988487:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177318.988489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177318.988493:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177318.988494:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177318.988498:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177318.988498:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177318.988503:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177318.988593:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177318.988597:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177318.988599:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177319.116561:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177319.116571:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177319.116574:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177319.116578:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177319.116582:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177319.116589:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11993 00000400:00000200:3.0:1786177319.116596:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177319.116603:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177319.116605:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177319.116609:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.116612:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177319.116615:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.116617:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177319.116621:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177319.116623:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177319.116626:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177319.116629:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177319.116631:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177319.116634:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177319.116637:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.116639:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177319.116641:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.116643:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177319.116646:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177319.116649:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177319.116651:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177319.116654:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177319.116657:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177319.116660:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.116662:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177319.116664:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177319.116673:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786177319.116677:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177319.116681:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177319.116683:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.116686:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177319.116689:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177319.116692:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177319.116699:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786177319.116705:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177319.116714:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177319.116724:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177319.116728:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177319.116734:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177319.116754:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177319.116757:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177319.116760:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177319.116761:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177319.116763:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177319.116766:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177319.116770:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11993 00000400:00000200:3.0:1786177319.116775:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177319.116776:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1786177319.116779:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:1.0:1786177319.116781:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1786177319.116784:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1786177319.116785:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177319.116793:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177319.116793:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.116798:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177319.116803:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177319.116807:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177319.116808:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:1.0:1786177319.116810:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177319.116812:0:191280:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:1.0:1786177319.116814:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1786177319.116815:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 02000000:00000001:1.0:1786177319.116817:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1786177319.116818:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1786177319.116820:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1786177319.116823:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00100000:1.0:1786177319.116824:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235602560:192.168.202.132@tcp:250:kworker.0 00000020:00000001:3.0:1786177319.116827:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:1.0:1786177319.116829:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1786177319.116831:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 02000000:00000001:1.0:1786177319.116833:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177319.116836:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177319.116837:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:1.0:1786177319.116838:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1786177319.116841:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177319.116844:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.116846:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177319.116848:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177319.116850:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177319.116853:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177319.116854:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177319.116858:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177319.116860:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.116862:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177319.116866:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177319.116868:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.116870:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177319.116873:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786177319.116876:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177319.116878:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177319.116880:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.116881:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177319.116883:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177319.116884:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177319.116888:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786177319.116891:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177319.116895:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177319.116902:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177319.116904:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177319.116907:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177319.116915:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177319.116917:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177319.116920:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177319.116922:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177319.116923:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177319.116926:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177319.116928:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11993 00000400:00000200:3.0:1786177319.116931:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177319.116934:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177319.116936:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177319.116938:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.116940:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177319.116942:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.116944:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177319.116946:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177319.116947:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177319.116948:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177319.116950:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177319.116952:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177319.116954:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177319.116957:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.116959:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177319.117035:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.117038:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177319.117040:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177319.117043:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177319.117045:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177319.117047:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177319.117050:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177319.117051:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.117053:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177319.117055:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177319.117057:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.117059:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177319.117062:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786177319.117065:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177319.117067:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177319.117068:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.117070:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177319.117071:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177319.117073:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177319.117075:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000010:3.0:1786177319.117076:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 02000000:00000001:1.0:1786177319.117079:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.117081:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177319.117086:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:3.0:1786177319.117089:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1786177319.117091:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177319.117099:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000001:3.0:1786177319.117100:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177319.117104:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1786177319.117105:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235602560, portal 25 00000100:00000001:1.0:1786177319.117109:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1786177319.117110:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117111:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280810880 : -110371428740736 : ffff9b9e26325180) 00000100:00000040:1.0:1786177319.117120:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177319.117125:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.117131:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1786177319.117131:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1786177319.117137:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177319.117138:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000010:1.0:1786177319.117139:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000001:3.0:1786177319.117141:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:1.0:1786177319.117142:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235602560, offset 0 00000100:00080000:3.0:1786177319.117144:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177319.117148:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1786177319.117149:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177319.117167:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11993 00000400:00000200:3.0:1786177319.117172:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177319.117175:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177319.117178:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177319.117181:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1786177319.117182:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000040:3.0:1786177319.117188:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.117191:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1786177319.117235:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177319.117265:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000040:3.0:1786177319.117266:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:1.0:1786177319.117270:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1786177319.117271:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:1.0:1786177319.117274:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000001:3.0:1786177319.117276:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000400:00000200:1.0:1786177319.117280:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426774:426774:256:4294967295] 192.168.202.132@tcp LPNI seq info [426736:426736:8:4294967295] 00000020:00000040:3.0:1786177319.117280:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177319.117283:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1786177319.117286:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000040:3.0:1786177319.117287:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177319.117291:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177319.117294:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000200:1.0:1786177319.117298:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000040:3.0:1786177319.117302:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.117305:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1786177319.117307:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000040:3.0:1786177319.117309:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000010:1.0:1786177319.117313:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000100:00000001:3.0:1786177319.117313:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177319.117319:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177319.117324:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:1.0:1786177319.117325:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177319.117327:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1786177319.117329:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 02000000:00000001:3.0:1786177319.117331:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177319.117335:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.1:1786177319.117336:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:3.0:1786177319.117339:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1786177319.117340:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000100:00000001:3.0:1786177319.117345:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000100:1.0:1786177319.117346:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177319.117349:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000008:00100000:3.0:1786177319.117350:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000200:1.0:1786177319.117354:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000001:3.0:1786177319.117355:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177319.117375:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177319.117381:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177319.117383:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177319.117387:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177319.117391:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117399:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177319.117403:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117410:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.117412:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177319.117418:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177319.117422:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177319.117425:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306669825 00000100:00000010:1.0:1786177319.117429:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c622a0. 00000100:00000001:1.0:1786177319.117434:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177319.117440:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306669825 00000100:00000010:1.0:1786177319.117441:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e08c62300. 00000100:00000001:1.0:1786177319.117557:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177319.117560:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177319.117562:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.117564:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177319.117568:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177319/real 1786177319] req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177319.117575:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177319.117576:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177319.117577:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177319.117579:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177319.117580:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177319.117582:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177319.117584:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177319.117590:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117593:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177319.117594:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.117596:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177319.117597:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177319.117599:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117603:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177319.117606:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117610:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177319.117612:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.117614:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177319.117615:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.117617:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177319.117619:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177319.117622:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177319.117626:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177319.117628:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177319.117632:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177319.117640:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235602560:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177319.117645:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.117647:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177319.117650:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325180 x1872930235602560/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117657:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177319.117660:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177319.117662:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177319.117663:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177319.117665:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177319.117666:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177319.117667:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177319.117670:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8400. 02000000:00000001:1.0:1786177319.117672:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177319.117673:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177319.117675:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325180. 00000100:00000001:1.0:1786177319.117677:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177319.117677:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.117679:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.117684:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177319.117686:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.117687:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177319.117688:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177319.117689:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177319.117691:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117696:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177319.117697:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177319.117699:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177319.117700:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177319.117701:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177319.117703:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602688:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177319.117705:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177319.117706:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177319.117707:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177319.117708:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177319.117710:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177319.117712:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177319.117714:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177319.117716:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177319.117718:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177319.117720:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235602688, portal 10 00000100:00000001:1.0:1786177319.117722:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177319.117723:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818048 : -110371428733568 : ffff9b9e26326d80) 00000100:00000040:1.0:1786177319.117727:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117731:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177319.117732:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177319.117733:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177319.117734:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235602688, offset 0 00000400:00000200:1.0:1786177319.117737:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177319.117739:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177319.117742:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177319.117744:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177319.117745:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177319.117747:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177319.117748:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177319.117750:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426775:426775:256:4294967295] 192.168.202.132@tcp LPNI seq info [426737:426737:8:4294967295] 00000400:00000200:1.0:1786177319.117752:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177319.117756:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177319.117760:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177319.117761:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177319.117764:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177319.117766:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786177319.117769:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177319.117770:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1786177319.117773:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177319.117774:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177319.117776:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177319.117778:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177319.117780:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177319.117781:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177319.117782:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177319.117784:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117789:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177319.117790:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117794:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.117795:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177319.117797:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177319.117800:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:1.0:1786177319.117801:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177319.117803:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306665988 00000100:00000001:1.0:1786177319.117804:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306669825 00000100:00000001:1.0:1786177319.117806:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306665988 00000100:00000001:1.0:1786177319.117807:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306669825 00000100:00000001:1.0:1786177319.117808:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177319.117809:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.117810:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177319.117812:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177319/real 1786177319] req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177319.117817:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177319.117818:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177319.117819:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177319.117820:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177319.117821:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177319.117822:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177319.117823:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177319.117828:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117831:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177319.117832:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.117833:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177319.117834:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177319.117836:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117840:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177319.117842:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117845:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177319.117847:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.117847:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177319.117848:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.117849:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177319.117850:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177319.117852:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177319.117853:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177319.117854:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177319.117857:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177319.117862:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602688:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177319.117864:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177319.117866:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117869:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177319.117870:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177319.117872:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177319.117873:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177319.117875:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177319.117877:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602816:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177319.117880:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177319.117881:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177319.117882:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177319.117884:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177319.117887:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177319.117888:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177319.117891:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177319.117893:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177319.117896:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177319.117899:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235602816, portal 10 00000100:00000001:1.0:1786177319.117901:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177319.117902:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280820736 : -110371428730880 : ffff9b9e26327800) 00000100:00000040:1.0:1786177319.117908:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117914:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177319.117916:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177319.117918:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177319.117920:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235602816, offset 0 00000400:00000200:1.0:1786177319.117923:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177319.117926:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177319.117928:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177319.117930:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177319.117931:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177319.117933:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177319.117934:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177319.117936:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426776:426776:256:4294967295] 192.168.202.132@tcp LPNI seq info [426738:426738:8:4294967295] 00000400:00000200:1.0:1786177319.117938:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177319.117942:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177319.117945:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177319.117946:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177319.117949:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177319.117951:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786177319.117953:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177319.117955:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1786177319.117957:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177319.117958:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177319.117960:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177319.117962:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177319.117964:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177319.117965:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177319.117966:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177319.117968:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117973:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177319.117974:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.117978:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.117979:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177319.117981:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177319.117983:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:1.0:1786177319.117984:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177319.117986:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306665988 00000100:00000001:1.0:1786177319.117987:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306669825 00000100:00000001:1.0:1786177319.117989:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306665988 00000100:00000001:1.0:1786177319.117990:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306669825 00000100:00000001:1.0:1786177319.117991:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177319.117992:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.117993:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177319.117995:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177319/real 1786177319] req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177319.118065:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177319.118067:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177319.118069:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177319.118071:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177319.118073:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177319.118076:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177319.118080:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177319.118087:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118091:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177319.118092:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118093:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177319.118094:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177319.118096:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118100:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177319.118102:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118106:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177319.118108:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118109:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177319.118110:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118111:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177319.118112:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177319.118114:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177319.118116:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177319.118117:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177319.118119:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177319.118123:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602816:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177319.118125:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118127:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177319.118128:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326d80 x1872930235602688/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118132:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177319.118133:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177319.118134:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:1.0:1786177319.118136:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177319.118137:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177319.118137:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177319.118139:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177319.118140:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:1.0:1786177319.118142:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177319.118142:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177319.118143:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326d80. 00000100:00000001:1.0:1786177319.118145:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177319.118146:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118147:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177319.118148:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327800 x1872930235602816/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118165:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177319.118166:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177319.118167:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177319.118168:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177319.118169:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177319.118170:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177319.118171:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177319.118172:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:1.0:1786177319.118173:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177319.118174:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177319.118174:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327800. 00000100:00000001:1.0:1786177319.118176:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177319.118177:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118178:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118180:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177319.118181:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.118182:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177319.118183:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.118438:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177319.118441:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786177319.118443:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177319.118445:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177319.118446:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.118447:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177319.118448:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177319.118450:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177319.118453:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786177319.118455:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177319.118460:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177319.118465:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177319.118467:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177319.118470:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177319.118484:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177319.118486:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177319.118489:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177319.118490:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177319.118491:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177319.118492:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177319.118494:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11993 00000400:00000200:3.0:1786177319.118496:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177319.118499:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177319.118501:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177319.118504:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.118505:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177319.118507:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.118509:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177319.118511:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177319.118512:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000001:1.0:1786177319.118513:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1786177319.118514:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1786177319.118517:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1786177319.118517:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177319.118519:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:1.0:1786177319.118522:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1786177319.118522:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177319.118526:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177319.118528:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177319.118531:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:1.0:1786177319.118532:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177319.118533:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177319.118536:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177319.118540:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177319.118543:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177319.118546:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177319.118550:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177319.118551:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1786177319.118552:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.118555:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.118555:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1786177319.118559:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:3.0:1786177319.118559:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 02000000:00000001:1.0:1786177319.118562:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1786177319.118563:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177319.118567:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177319.118567:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177319.118570:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000100:00100000:1.0:1786177319.118573:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602944:192.168.202.132@tcp:8:kworker.0 00000020:00000040:3.0:1786177319.118573:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177319.118577:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1786177319.118579:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1786177319.118580:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177319.118582:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1786177319.118582:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177319.118584:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177319.118585:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177319.118586:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177319.118587:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1786177319.118590:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 02000000:00000010:1.0:1786177319.118591:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae800. 00000100:00000001:3.0:1786177319.118593:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177319.118595:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177319.118599:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1786177319.118602:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177319.118607:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177319.118611:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000001:3.0:1786177319.118616:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1786177319.118618:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235602944, portal 4 00000100:00000001:3.0:1786177319.118618:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177319.118623:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118624:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177319.118627:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280814464 : -110371428737152 : ffff9b9e26325f80) 00000100:00000001:3.0:1786177319.118634:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177319.118637:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12003) 00000100:00000040:1.0:1786177319.118638:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118647:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177319.118651:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177319.118654:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177319.118656:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235602944, offset 0 00000400:00000200:1.0:1786177319.118661:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177319.118664:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177319.118666:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177319.118668:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177319.118670:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177319.118671:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177319.118673:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177319.118675:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426777:426777:256:4294967295] 192.168.202.132@tcp LPNI seq info [426739:426739:8:4294967295] 00000400:00000200:1.0:1786177319.118677:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177319.118681:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177319.118683:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177319.118686:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177319.118688:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177319.118690:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786177319.118692:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177319.118694:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1786177319.118696:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177319.118697:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177319.118699:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177319.118701:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177319.118704:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177319.118705:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177319.118706:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177319.118708:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118713:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177319.118715:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118719:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.118720:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177319.118721:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177319.118723:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:1.0:1786177319.118726:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177319.118727:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306665989 00000100:00000001:1.0:1786177319.118728:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306669825 00000100:00000001:1.0:1786177319.118730:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306665989 00000100:00000001:1.0:1786177319.118731:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306669825 00000100:00000001:1.0:1786177319.118732:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177319.118733:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.118734:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177319.118736:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177319/real 1786177319] req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177319.118740:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177319.118741:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177319.118742:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177319.118743:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177319.118744:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177319.118745:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177319.118747:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177319.118755:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118759:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177319.118760:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118761:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177319.118762:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177319.118764:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118767:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177319.118769:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118773:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177319.118774:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118775:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177319.118775:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118776:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177319.118777:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177319.118779:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177319.118780:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177319.118782:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177319.118784:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177319.118789:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235602944:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177319.118791:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118792:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177319.118794:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325f80 x1872930235602944/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118801:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177319.118802:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177319.118803:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177319.118805:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177319.118805:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177319.118806:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177319.118807:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177319.118808:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:1.0:1786177319.118810:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177319.118810:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177319.118811:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325f80. 00000100:00000001:1.0:1786177319.118813:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177319.118813:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118815:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118817:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177319.118818:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.118819:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177319.118820:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177319.118821:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177319.118823:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118827:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177319.118828:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177319.118829:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1786177319.118830:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177319.118831:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177319.118832:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603072:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177319.118834:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177319.118835:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177319.118836:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177319.118837:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1786177319.118838:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177319.118839:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1786177319.118841:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:1.0:1786177319.118842:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e068421b8. 00000400:00000010:1.0:1786177319.118844:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027ef68. 00000100:00000200:1.0:1786177319.118846:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235603072, portal 4 00000100:00000001:1.0:1786177319.118847:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1786177319.118848:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280819840 : -110371428731776 : ffff9b9e26327480) 00000100:00000040:1.0:1786177319.118851:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118855:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177319.118856:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:1.0:1786177319.118858:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e1027e000. 00000100:00000200:1.0:1786177319.118859:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235603072, offset 0 00000400:00000200:1.0:1786177319.118861:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177319.118863:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177319.118864:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:1.0:1786177319.118866:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177319.118867:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177319.118869:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1786177319.118870:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:1.0:1786177319.118872:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426778:426778:256:4294967295] 192.168.202.132@tcp LPNI seq info [426740:426740:8:4294967295] 00000400:00000200:1.0:1786177319.118875:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177319.118881:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:1.0:1786177319.118885:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177319.118888:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1142e600. 00000800:00000200:1.0:1786177319.118892:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.0:1786177319.118894:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1786177319.118898:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:1.1:1786177319.118900:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1786177319.118904:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:1.0:1786177319.118905:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1142e600. 00000400:00000200:1.0:1786177319.118908:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1786177319.118912:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177319.118914:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027e000 00000400:00000010:1.0:1786177319.118916:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027e000. 00000100:00000001:1.0:1786177319.118918:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177319.118922:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118926:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177319.118928:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177344 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118932:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.118933:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177319.118935:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177319.118937:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:1.0:1786177319.118939:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177319.118940:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306665991 00000100:00000001:1.0:1786177319.118942:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306669825 00000100:00000001:1.0:1786177319.118943:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306665991 00000100:00000001:1.0:1786177319.118944:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306669825 00000100:00000001:1.0:1786177319.118945:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177319.118946:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.118947:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177319.118949:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177319/real 1786177319] req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177319.118954:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177319.118954:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177319.118955:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e068421b8. 00000400:00000200:1.0:1786177319.118957:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e1027ef68 00000400:00000010:1.0:1786177319.118958:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e1027ef68. 00000100:00000001:1.0:1786177319.118959:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177319.118961:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177319.118965:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118972:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177319.118973:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118974:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177319.118975:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177319.118977:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118980:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177319.118982:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.118986:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177319.118987:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118988:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177319.118989:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.118989:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177319.118990:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177319.118992:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177319.118993:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177319.118994:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177319.119172:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177319.119179:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603072:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177319.119183:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.119184:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177319.119187:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327480 x1872930235603072/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177344 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177319.119883:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177319.119885:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177319.119888:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:1.0:1786177319.119891:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177319.119892:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177319.119894:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177319.119896:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177319.119898:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:1.0:1786177319.119900:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177319.119901:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177319.119903:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327480. 00000100:00000001:1.0:1786177319.119905:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177319.119907:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.119909:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177319.119913:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177319.119914:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177319.119915:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177319.119916:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177319.628452:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177319.628458:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177319.628460:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177319.628465:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177319.628468:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177319.628471:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177319.628473:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177319.628475:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177319.628477:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177320.012482:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177320.012487:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177320.012490:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177320.012492:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177320.012496:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177320.012499:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1786177320.012501:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177320.012502:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177320.012505:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177320.012509:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177320.012510:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177320.012512:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177320.012515:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177320.012519:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177320.012521:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177320.012573:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177320.012575:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177320.012578:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177320.652429:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177320.652437:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177320.652440:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177320.652445:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177320.652448:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177320.652452:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177320.652455:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177320.652457:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177320.652460:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177321.036431:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177321.036438:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177321.036441:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177321.036447:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177321.036450:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177321.036450:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177321.036454:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177321.036456:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177321.036458:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177321.036463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177321.036466:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177321.036467:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177321.036471:0:206592:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177321.036475:0:206592:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177321.036480:0:206592:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177321.036484:0:206592:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177321.036487:0:206592:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177321.036489:0:206592:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177321.036492:0:206592:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177321.036496:0:206592:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177321.036498:0:206592:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177321.036553:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177321.036557:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177321.036560:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177321.676511:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177321.676520:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177321.676522:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177321.676527:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177321.676531:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177321.676535:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177321.676537:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177321.676539:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177321.676540:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177322.060436:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177322.060441:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177322.060443:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177322.060447:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177322.060449:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177322.060452:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177322.060454:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177322.060456:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177322.060458:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177322.062238:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177322.062243:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177322.062249:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177322.062250:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177322.062254:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177322.062255:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177322.062343:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177322.062347:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177322.062350:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177322.700454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177322.700462:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177322.700465:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177322.700470:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177322.700474:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177322.700477:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177322.700479:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177322.700481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177322.700485:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177323.020450:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306669825, at 4306669888 00000100:00000040:1.0:1786177323.020450:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306669825, at 4306669888 00000100:00000010:1.0:1786177323.020464:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c62300. 00000100:00000010:0.0:1786177323.020465:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e08c622a0. 00000100:00000001:3.0:1786177323.084428:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177323.084436:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177323.084437:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177323.084438:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177323.084442:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177323.084445:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177323.084448:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177323.084450:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177323.084451:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177323.084455:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177323.084457:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177323.084459:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177323.084461:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177323.084463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177323.084467:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177323.084507:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177323.084510:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177323.084512:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177323.724472:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177323.724481:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177323.724483:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177323.724490:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177323.724495:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177323.724499:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177323.724502:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177323.724504:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177323.724507:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177324.108484:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177324.108493:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177324.108495:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177324.108501:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177324.108505:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177324.108509:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177324.108512:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177324.108515:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177324.108519:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177324.110248:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177324.110256:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177324.110258:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177324.110262:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177324.110267:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177324.110269:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177324.110271:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177324.110273:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177324.110278:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177324.110279:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177324.110282:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177324.110284:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177324.110285:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177324.110287:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177324.110288:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177324.110289:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177324.110291:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177324.110293:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177324.111174:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177324.111179:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177324.111182:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177324.236466:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177324.236476:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177324.236478:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177324.236482:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177324.236487:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177324.236494:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 11998 00000400:00000200:1.0:1786177324.236503:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177324.236508:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177324.236512:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177324.236516:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.236519:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177324.236522:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.236524:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177324.236528:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177324.236530:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177324.236532:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177324.236535:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177324.236538:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177324.236540:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177324.236543:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.236545:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177324.236547:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.236548:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177324.236551:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177324.236554:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177324.236556:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177324.236559:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177324.236562:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177324.236565:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177324.236566:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177324.236569:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177324.236577:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328baa00. 00000020:00000040:1.0:1786177324.236580:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177324.236584:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177324.236586:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177324.236589:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177324.236592:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177324.236596:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177324.236605:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac400. 00000100:00000001:1.0:1786177324.236612:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177324.236621:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177324.236631:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177324.236635:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177324.236640:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ebce398] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177324.236661:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177324.236668:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177324.236672:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177324.236674:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177324.236675:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177324.236679:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177324.236682:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11998 00000400:00000200:1.0:1786177324.236685:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177324.236688:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1786177324.236689:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:0.0:1786177324.236695:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1786177324.236695:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1786177324.236700:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177324.236700:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.236704:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177324.236707:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.236710:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000040:0.0:1786177324.236712:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177324.236713:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177324.236715:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177324.236718:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177324.236721:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177324.236726:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:0.0:1786177324.236731:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1786177324.236731:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:0.0:1786177324.236735:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177324.236738:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:0.0:1786177324.236741:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1786177324.236741:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1786177324.236746:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1786177324.236746:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.236751:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:0.0:1786177324.236753:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177324.236756:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00100000:0.0:1786177324.236760:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235603200:192.168.202.132@tcp:250:kworker.0 00000100:00080000:1.0:1786177324.236763:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177324.236767:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1786177324.236768:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177324.236771:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177324.236773:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177324.236777:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177324.236777:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177324.236783:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1786177324.236783:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177324.236790:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:1.0:1786177324.236794:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:0.0:1786177324.236795:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1786177324.236801:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1786177324.236804:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000008:00000001:1.0:1786177324.236807:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1786177324.236810:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000100:00000001:1.0:1786177324.236810:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177324.236818:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bad80. 00000400:00000010:0.0:1786177324.236820:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000020:00000040:1.0:1786177324.236821:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177324.236827:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177324.236828:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1786177324.236830:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235603200, portal 25 00000100:00000001:1.0:1786177324.236831:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177324.236833:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1786177324.236836:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1786177324.236837:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.236839:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487992832 : -110371221558784 : ffff9b9e328baa00) 02000000:00000010:1.0:1786177324.236844:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac800. 00000100:00000040:0.0:1786177324.236846:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177324.236849:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177324.236859:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177324.236860:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177324.236869:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177324.236873:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000001:1.0:1786177324.236875:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:0.0:1786177324.236878:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235603200, offset 0 00000100:00000001:1.0:1786177324.236879:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177324.236886:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b748f214] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1786177324.236890:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177324.236900:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177324.236901:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177324.236904:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177324.236908:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1786177324.236909:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00000001:1.0:1786177324.236911:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177324.236915:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1786177324.236916:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1786177324.236919:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1786177324.236922:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00080000:1.0:1786177324.236926:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11998 00000400:00000200:0.0:1786177324.236929:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177324.236934:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177324.236938:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426779:426779:256:4294967295] 192.168.202.132@tcp LPNI seq info [426741:426741:8:4294967295] 00000400:00000200:0.0:1786177324.236943:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177324.236951:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000400:00000200:1.0:1786177324.236958:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:0.0:1786177324.236959:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177324.236963:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177324.236966:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000010:0.0:1786177324.236967:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000100:00000040:1.0:1786177324.236968:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.236972:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177324.236973:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.236977:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:0.0:1786177324.236978:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:1.0:1786177324.236979:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177324.236982:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177324.236983:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:0.0:1786177324.236984:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000020:00000040:1.0:1786177324.236985:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177324.236990:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177324.236994:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000800:00000200:0.1:1786177324.236995:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:1.0:1786177324.237001:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:0.1:1786177324.237001:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000100:00000001:1.0:1786177324.237007:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:0.0:1786177324.237011:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000040:1.0:1786177324.237012:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000010:0.0:1786177324.237016:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000100:00000001:1.0:1786177324.237017:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177324.237022:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:0.0:1786177324.237024:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:1.0:1786177324.237027:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:0.0:1786177324.237033:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1786177324.237033:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177324.237038:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1786177324.237039:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 02000000:00000001:1.0:1786177324.237042:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1786177324.237043:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 02000000:00000001:1.0:1786177324.237046:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237050:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1786177324.237050:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177324.237055:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000200:0.0:1786177324.237056:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1786177324.237059:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177324.237063:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177324.237070:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bbb80. 00000100:00000001:0.0:1786177324.237073:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1786177324.237074:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000040:0.0:1786177324.237078:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177324.237080:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177324.237083:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177324.237087:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177324.237090:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1786177324.237094:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177324.237094:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237098:0:2392:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:1.0:1786177324.237102:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac000. 00000100:00000001:1.0:1786177324.237118:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237121:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177324.237125:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1786177324.237129:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177324.237134:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306674944 00000100:00000010:0.0:1786177324.237140:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5840. 00000100:00000001:1.0:1786177324.237148:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177324.237149:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177324.237152:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237157:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306674944 00000100:00000040:1.0:1786177324.237159:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3691ff3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1786177324.237163:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5000. 00000100:00000001:0.0:1786177324.237169:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1786177324.237174:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237175:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1786177324.237179:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1786177324.237180:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237186:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1786177324.237187:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177324.237190:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1786177324.237193:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177324/real 1786177324] req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177324.237194:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177324.237200:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177324.237205:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11998 00000400:00000001:0.0:1786177324.237210:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177324.237214:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1786177324.237214:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177324.237219:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:1.0:1786177324.237221:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:0.0:1786177324.237224:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000100:00080000:1.0:1786177324.237226:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1786177324.237227:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000040:1.0:1786177324.237232:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177324.237233:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1786177324.237237:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1786177324.237239:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177324.237243:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.237249:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177324.237255:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177324.237259:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000200:0.0:1786177324.237260:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177324.237263:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177324.237267:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177324.237273:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:0.0:1786177324.237274:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177324.237276:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1786177324.237278:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:0.0:1786177324.237280:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177324.237284:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177324.237287:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00100000:0.0:1786177324.237290:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177324.237292:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177324.237296:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.237302:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177324.237307:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177324.237308:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000040:0.0:1786177324.237315:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177324.237315:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177324.237321:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177324.237326:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177324.237330:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177324.237331:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1786177324.237335:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237337:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177324.237339:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177324.237341:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177324.237343:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1786177324.237343:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1786177324.237347:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00000001:1.0:1786177324.237349:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237350:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177324.237353:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177324.237355:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177324.237380:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177324.237385:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000010:1.0:1786177324.237389:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8000. 00000100:00000040:0.0:1786177324.237390:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177324.237396:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177324.237403:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177324.237407:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177324.237408:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235603200:192.168.202.132@tcp:250:kworker.0 00000100:00000001:1.0:1786177324.237412:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1786177324.237415:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177324.237415:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1786177324.237419:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1786177324.237420:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177324.237424:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328baa00 x1872930235603200/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177324.237427:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261acc00. 00000100:00000001:1.0:1786177324.237432:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237437:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177324.237441:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:1.0:1786177324.237441:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:0.0:1786177324.237445:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177324.237451:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1786177324.237451:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:0.0:1786177324.237455:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:1.0:1786177324.237455:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1786177324.237459:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000100:00000040:1.0:1786177324.237461:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ab5f277] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1786177324.237463:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177324.237467:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac400. 00000100:00000001:1.0:1786177324.237469:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177324.237472:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:1.0:1786177324.237472:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177324.237475:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:0.0:1786177324.237476:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1786177324.237479:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000010:0.0:1786177324.237480:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328baa00. 00000100:00080000:1.0:1786177324.237482:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177324.237486:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177324.237486:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1786177324.237489:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1786177324.237492:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 11998 00000100:00000001:0.0:1786177324.237494:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1786177324.237500:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177324.237503:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:1.0:1786177324.237505:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:0.0:1786177324.237507:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177324.237510:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1786177324.237512:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177324.237515:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177324.237518:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1786177324.237518:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.237522:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177324.237526:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177324.237526:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177324.237529:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177324.237533:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177324.237535:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177324.237538:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177324.237541:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:0.0:1786177324.237543:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1786177324.237544:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:0.0:1786177324.237547:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177324.237547:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 02000000:00000001:0.0:1786177324.237552:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1786177324.237553:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:0.0:1786177324.237556:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177324.237556:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177324.237558:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:0.0:1786177324.237562:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177324.237562:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177324.237567:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00100000:0.0:1786177324.237570:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603328:192.168.202.132@tcp:38:kworker.0 00000100:00080000:1.0:1786177324.237571:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177324.237575:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237578:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1786177324.237578:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177324.237582:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177324.237583:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1786177324.237585:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177324.237587:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177324.237587:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177324.237590:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 02000000:00000001:0.0:1786177324.237591:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:1.0:1786177324.237592:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177324.237595:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177324.237597:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 00000100:00000010:1.0:1786177324.237598:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9500. 02000000:00000001:0.0:1786177324.237601:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177324.237602:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177324.237607:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177324.237609:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177324.237610:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177324.237611:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177324.237614:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1786177324.237616:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 02000000:00000001:1.0:1786177324.237616:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177324.237620:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261adc00. 00000400:00000010:0.0:1786177324.237622:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000001:1.0:1786177324.237624:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1786177324.237628:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235603328, portal 10 00000100:00100000:1.0:1786177324.237630:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177324.237633:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177324.237636:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487993728 : -110371221557888 : ffff9b9e328bad80) 00000100:00000001:1.0:1786177324.237640:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177324.237643:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177324.237649:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000cd924b3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786177324.237650:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177324.237667:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237670:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177324.237672:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12008) 00000100:00000040:0.0:1786177324.237676:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177324.237679:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177324.237682:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235603328, offset 0 00000400:00000200:0.0:1786177324.237686:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177324.237691:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177324.237695:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177324.237697:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177324.237700:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177324.237702:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177324.237704:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177324.237708:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426780:426780:256:4294967295] 192.168.202.132@tcp LPNI seq info [426742:426742:8:4294967295] 00000400:00000200:0.0:1786177324.237712:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177324.237718:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177324.237722:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177324.237726:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177324.237730:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177324.237733:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1786177324.237737:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177324.237740:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1786177324.237743:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177324.237745:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177324.237748:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177324.237752:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177324.237754:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177324.237756:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177324.237758:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177324.237762:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.237770:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177324.237773:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.237780:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237782:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177324.237784:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177324.237788:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177324.237791:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177324.237792:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306671108 00000100:00000001:0.0:1786177324.237795:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306674944 00000100:00000001:0.0:1786177324.237797:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306671108 00000100:00000001:0.0:1786177324.237800:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306674944 00000100:00000001:0.0:1786177324.237802:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177324.237804:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.237806:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177324.237810:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177324/real 1786177324] req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177324.237824:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177324.237825:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177324.237827:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177324.237829:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177324.237830:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177324.237832:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177324.237835:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177324.237844:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.237849:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177324.237850:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.237852:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177324.237853:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177324.237857:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.237864:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177324.237867:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.237874:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177324.237876:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.237878:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177324.237879:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.237880:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177324.237882:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177324.237885:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177324.237887:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177324.237889:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177324.237893:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177324.237901:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603328:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177324.237904:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177324.237907:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.237913:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177324.237915:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177324.237916:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177324.237918:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177324.237920:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177324.237922:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603456:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177324.237925:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177324.237927:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177324.237929:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177324.237930:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177324.237934:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177324.237936:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177324.237939:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177324.237941:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177324.237944:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177324.237946:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235603456, portal 10 00000100:00000001:0.0:1786177324.237948:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177324.237949:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487997312 : -110371221554304 : ffff9b9e328bbb80) 00000100:00000040:0.0:1786177324.237955:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.237961:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177324.237963:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177324.237965:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177324.237968:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235603456, offset 0 00000400:00000200:0.0:1786177324.237972:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177324.237975:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177324.237978:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177324.237981:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177324.237983:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177324.237986:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177324.237988:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177324.237991:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426781:426781:256:4294967295] 192.168.202.132@tcp LPNI seq info [426743:426743:8:4294967295] 00000400:00000200:0.0:1786177324.237995:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177324.238001:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177324.238005:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177324.238008:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177324.238011:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177324.238014:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1786177324.238018:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177324.238020:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1786177324.238024:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177324.238026:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177324.238028:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177324.238031:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177324.238034:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177324.238036:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177324.238038:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177324.238041:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238048:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177324.238051:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238057:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.238059:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177324.238061:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177324.238064:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177324.238066:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177324.238069:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306671108 00000100:00000001:0.0:1786177324.238071:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306674944 00000100:00000001:0.0:1786177324.238073:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306671108 00000100:00000001:0.0:1786177324.238075:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306674944 00000100:00000001:0.0:1786177324.238077:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177324.238079:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.238081:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177324.238085:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177324/real 1786177324] req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177324.238093:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177324.238095:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177324.238096:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177324.238098:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177324.238100:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177324.238102:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177324.238116:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177324.238125:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238132:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177324.238133:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238135:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177324.238136:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177324.238139:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238146:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177324.238149:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238155:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177324.238158:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238160:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177324.238161:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238163:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177324.238165:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177324.238167:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177324.238170:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177324.238172:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177324.238176:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177324.238183:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603456:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177324.238186:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177324.238189:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238195:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177324.238196:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177324.238198:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177324.238200:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177324.238202:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177324.238205:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603584:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177324.238208:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177324.238210:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177324.238212:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177324.238214:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177324.238217:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177324.238219:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177324.238222:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177324.238224:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177324.238227:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177324.238230:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235603584, portal 4 00000100:00000001:0.0:1786177324.238232:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177324.238233:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982080 : -110371221569536 : ffff9b9e328b8000) 00000100:00000040:0.0:1786177324.238239:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238245:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177324.238247:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177324.238249:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177324.238251:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235603584, offset 0 00000400:00000200:0.0:1786177324.238255:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177324.238258:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177324.238262:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177324.238264:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177324.238267:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177324.238269:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177324.238271:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177324.238274:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426782:426782:256:4294967295] 192.168.202.132@tcp LPNI seq info [426744:426744:8:4294967295] 00000400:00000200:0.0:1786177324.238278:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177324.238284:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177324.238288:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177324.238291:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177324.238294:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177324.238297:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1786177324.238301:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177324.238304:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1786177324.238307:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177324.238309:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177324.238312:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177324.238315:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177324.238317:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177324.238319:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177324.238321:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177324.238324:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238337:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177324.238340:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238347:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.238348:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177324.238351:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177324.238353:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177324.238374:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177324.238376:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306671109 00000100:00000001:0.0:1786177324.238379:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306674944 00000100:00000001:0.0:1786177324.238381:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306671109 00000100:00000001:0.0:1786177324.238383:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306674944 00000100:00000001:0.0:1786177324.238385:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177324.238387:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.238388:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177324.238392:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177324/real 1786177324] req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177324.238399:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177324.238401:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177324.238402:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177324.238405:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177324.238407:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177324.238409:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177324.238412:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177324.238420:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238426:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177324.238428:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238430:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177324.238431:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177324.238434:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238440:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177324.238444:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238450:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177324.238452:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238454:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177324.238455:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238456:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177324.238458:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177324.238460:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177324.238462:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177324.238464:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177324.238468:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177324.238476:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603584:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177324.238479:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238481:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177324.238484:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bad80 x1872930235603328/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238490:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177324.238492:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177324.238494:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177324.238497:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177324.238498:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177324.238501:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177324.238503:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177324.238504:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:0.0:1786177324.238506:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177324.238508:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177324.238509:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bad80. 00000100:00000001:0.0:1786177324.238512:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177324.238514:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238515:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177324.238519:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bbb80 x1872930235603456/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238525:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177324.238526:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177324.238528:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177324.238529:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177324.238530:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177324.238531:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177324.238533:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177324.238534:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:0.0:1786177324.238536:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177324.238537:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177324.238539:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bbb80. 00000100:00000001:0.0:1786177324.238541:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177324.238542:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238544:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177324.238547:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8000 x1872930235603584/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238553:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177324.238554:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177324.238555:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177324.238557:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177324.238559:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177324.238560:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177324.238562:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177324.238563:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:0.0:1786177324.238564:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177324.238566:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177324.238567:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8000. 00000100:00000001:0.0:1786177324.238569:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177324.238570:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238572:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238575:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177324.238576:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.238578:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177324.238579:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177324.238580:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177324.238584:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238591:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177324.238592:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177324.238594:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177324.238596:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177324.238598:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177324.238600:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603712:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177324.238603:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177324.238605:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177324.238606:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177324.238608:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177324.238611:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177324.238612:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177324.238616:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177324.238618:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177324.238621:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177324.238623:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235603712, portal 4 00000100:00000001:0.0:1786177324.238625:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177324.238626:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487987456 : -110371221564160 : ffff9b9e328b9500) 00000100:00000040:0.0:1786177324.238632:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238637:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177324.238640:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177324.238642:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177324.238645:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235603712, offset 0 00000400:00000200:0.0:1786177324.238648:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177324.238652:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177324.238656:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177324.238659:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177324.238662:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177324.238664:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177324.238666:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177324.238669:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426783:426783:256:4294967295] 192.168.202.132@tcp LPNI seq info [426745:426745:8:4294967295] 00000400:00000200:0.0:1786177324.238673:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177324.238679:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177324.238683:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177324.238686:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177324.238689:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177324.238692:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1786177324.238696:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177324.238698:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 43 secs later) 00000800:00000100:0.0:1786177324.238702:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177324.238704:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177324.238706:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177324.238709:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177324.238711:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177324.238713:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177324.238715:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177324.238719:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238726:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177324.238730:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177349 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238736:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.238737:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177324.238740:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177324.238743:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177324.238746:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177324.238748:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306671111 00000100:00000001:0.0:1786177324.238750:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306674944 00000100:00000001:0.0:1786177324.238752:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306671111 00000100:00000001:0.0:1786177324.238755:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306674944 00000100:00000001:0.0:1786177324.238756:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177324.238758:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.238760:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177324.238764:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177324/real 1786177324] req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177324.238770:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177324.238771:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177324.238773:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177324.238775:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177324.238776:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177324.238778:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177324.238781:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177324.238789:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238795:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177324.238796:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238798:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177324.238799:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177324.238808:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238814:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177324.238818:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238826:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177324.238828:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238830:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177324.238831:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238833:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177324.238834:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177324.238837:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177324.238839:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177324.238841:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177324.238845:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177324.238853:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603712:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177324.238856:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238858:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177324.238861:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9500 x1872930235603712/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177349 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177324.238866:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177324.238868:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177324.238870:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177324.238872:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177324.238874:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177324.238875:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177324.238877:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177324.238878:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261adc00. 02000000:00000001:0.0:1786177324.238880:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177324.238882:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177324.238883:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9500. 00000100:00000001:0.0:1786177324.238885:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177324.238886:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238889:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177324.238891:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177324.238893:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177324.238894:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177324.238896:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177324.748655:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177324.748665:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177324.748668:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177324.748675:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177324.748680:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177324.748694:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177324.748698:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177324.748701:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177324.748705:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177325.132454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177325.132457:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177325.132461:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177325.132462:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177325.132466:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177325.132467:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177325.132471:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177325.132473:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177325.132474:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177325.132475:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177325.132480:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177325.132484:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177325.132486:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177325.132489:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177325.132491:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177325.132543:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177325.132547:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177325.132551:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177325.772448:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177325.772454:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177325.772456:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177325.772461:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177325.772466:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177325.772468:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177325.772470:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177325.772472:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177325.772481:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177326.156465:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177326.156471:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177326.156474:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177326.156476:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177326.156478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177326.156487:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177326.156489:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177326.156493:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177326.156496:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177326.156496:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177326.156503:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177326.156508:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177326.156511:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177326.156514:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177326.156518:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177326.156628:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177326.156636:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177326.156642:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177326.796532:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177326.796544:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177326.796547:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177326.796554:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177326.796559:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177326.796563:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177326.796566:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177326.796568:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177326.796572:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177327.180484:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177327.180491:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177327.180492:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177327.180495:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177327.180498:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177327.180505:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177327.180507:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177327.180508:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177327.180513:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177327.180514:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177327.180516:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177327.180517:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177327.180521:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177327.180524:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177327.180527:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177327.180591:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177327.180595:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177327.180598:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177327.820476:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177327.820487:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177327.820490:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177327.820497:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177327.820502:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177327.820506:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177327.820510:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177327.820514:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177327.820518:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177328.140488:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306674944, at 4306675008 00000100:00000040:1.0:1786177328.140489:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306674944, at 4306675008 00000100:00000010:0.0:1786177328.140513:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5840. 00000100:00000010:1.0:1786177328.140515:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5000. 00000100:00000001:3.0:1786177328.204510:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177328.204520:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177328.204523:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177328.204530:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177328.204546:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177328.204551:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177328.204554:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177328.204558:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177328.204562:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177328.206151:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177328.206157:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177328.206162:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177328.206164:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177328.206167:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177328.206169:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177328.207139:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177328.207145:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177328.207149:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177328.844451:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177328.844459:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177328.844462:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177328.844468:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177328.844471:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177328.844475:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177328.844477:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177328.844480:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177328.844484:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177329.228489:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177329.228495:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177329.228498:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177329.228501:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177329.228501:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177329.228508:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177329.228510:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177329.228511:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177329.228516:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177329.228516:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177329.228519:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177329.228521:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177329.228524:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177329.228526:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177329.228529:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177329.228596:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177329.228601:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177329.228606:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177329.356514:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177329.356529:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177329.356532:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177329.356537:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177329.356542:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177329.356552:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12003 00000400:00000200:2.0:1786177329.356562:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177329.356570:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177329.356573:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177329.356576:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177329.356580:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177329.356583:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177329.356586:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177329.356590:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177329.356593:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177329.356596:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177329.356599:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177329.356602:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177329.356604:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177329.356608:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177329.356610:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177329.356611:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177329.356613:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177329.356617:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177329.356620:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177329.356623:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177329.356625:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177329.356629:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177329.356632:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.356635:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177329.356637:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177329.356648:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 00000020:00000040:2.0:1786177329.356652:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177329.356657:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177329.356659:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.356662:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177329.356666:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177329.356678:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177329.356691:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177329.356698:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177329.356711:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177329.356722:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177329.356727:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177329.356731:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177329.356759:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177329.356763:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177329.356766:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177329.356768:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177329.356770:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177329.356773:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177329.356775:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12003 00000400:00000200:2.0:1786177329.356779:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177329.356782:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177329.356785:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177329.356788:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177329.356790:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177329.356791:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177329.356793:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177329.356794:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177329.356795:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177329.356797:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177329.356798:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177329.356799:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177329.356801:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177329.356804:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177329.356805:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177329.356808:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177329.356810:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177329.356811:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177329.356813:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177329.356814:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177329.356816:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177329.356821:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177329.356823:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.356825:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177329.356827:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1786177329.356829:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177329.356833:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.356835:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.356837:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:2.0:1786177329.356840:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000100:00000001:0.0:1786177329.356842:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1786177329.356843:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177329.356848:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177329.356849:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.356850:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177329.356852:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177329.356854:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177329.356859:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1786177329.356859:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786177329.356868:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177329.356882:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177329.356885:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177329.356889:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177329.356899:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177329.356903:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177329.356905:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1786177329.356909:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.356910:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177329.356917:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235603840:192.168.202.132@tcp:250:kworker.0 00000100:00000040:2.0:1786177329.356920:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.356927:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177329.356932:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177329.356936:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.356940:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177329.356941:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1786177329.356947:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177329.356953:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000010:0.0:1786177329.356954:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:2.0:1786177329.356959:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1786177329.356960:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177329.356966:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177329.356970:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177329.356975:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12003 00000400:00000200:0.0:1786177329.356980:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177329.356984:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177329.356988:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000200:2.0:1786177329.356993:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177329.356996:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1786177329.356998:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000040:2.0:1786177329.357001:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177329.357004:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177329.357006:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:0.0:1786177329.357007:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235603840, portal 25 00000100:00000001:2.0:1786177329.357009:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177329.357016:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1786177329.357016:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177329.357019:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000100:00000001:0.0:1786177329.357020:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469626368 : -110371239925248 : ffff9b9e31736a00) 00000020:00000001:2.0:1786177329.357022:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177329.357030:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000100:00000040:0.0:1786177329.357036:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177329.357037:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177329.357045:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177329.357067:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177329.357068:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1786177329.357070:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177329.357081:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000040:0.0:1786177329.357084:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177329.357090:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:0.0:1786177329.357091:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177329.357097:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235603840, offset 0 00000020:00000040:2.0:1786177329.357098:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177329.357108:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:0.0:1786177329.357117:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177329.357123:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177329.357129:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1786177329.357130:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177329.357135:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1786177329.357142:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 02000000:00000001:2.0:1786177329.357143:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.357152:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:0.0:1786177329.357155:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00100000:2.0:1786177329.357157:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1786177329.357163:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000008:00000001:2.0:1786177329.357165:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177329.357170:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1786177329.357171:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177329.357180:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000010:2.0:1786177329.357181:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000020:00000040:2.0:1786177329.357189:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000200:0.0:1786177329.357197:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426784:426784:256:4294967295] 192.168.202.132@tcp LPNI seq info [426746:426746:8:4294967295] 02000000:00000001:2.0:1786177329.357197:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177329.357203:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.357207:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1786177329.357208:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177329.357214:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177329.357218:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177329.357226:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 02000000:00000010:2.0:1786177329.357228:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000800:00000200:0.0:1786177329.357235:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:2.0:1786177329.357235:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1786177329.357244:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000100:00100000:2.0:1786177329.357246:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177329.357256:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:0.0:1786177329.357261:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177329.357261:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1786177329.357270:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00000040:2.0:1786177329.357271:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1786177329.357286:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177329.357294:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1786177329.357297:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1786177329.357304:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00080000:2.0:1786177329.357305:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:0.0:1786177329.357309:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000100:00080000:2.0:1786177329.357312:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177329.357316:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1786177329.357317:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1786177329.357319:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1786177329.357327:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1786177329.357327:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177329.357336:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12003 00000400:00000200:0.0:1786177329.357337:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177329.357343:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000400:00000200:2.0:1786177329.357349:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177329.357350:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177329.357400:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177329.357409:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:0.0:1786177329.357414:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1786177329.357415:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177329.357426:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177329.357426:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177329.357434:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177329.357445:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177329.357449:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.357450:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177329.357453:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1786177329.357459:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177329.357462:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000100:00000001:0.0:1786177329.357464:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177329.357466:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177329.357471:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000400:00000200:0.0:1786177329.357472:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:2.0:1786177329.357477:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:0.0:1786177329.357479:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306680065 00000020:00000001:2.0:1786177329.357485:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000010:0.0:1786177329.357489:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5840. 00000100:00000040:2.0:1786177329.357495:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177329.357499:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177329.357500:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177329.357507:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177329.357509:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306680065 00000100:00000001:2.0:1786177329.357511:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000010:0.0:1786177329.357515:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d56c0. 00000020:00000040:2.0:1786177329.357518:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:0.0:1786177329.357522:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1786177329.357524:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177329.357528:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177329.357529:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177329.357537:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1786177329.357538:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.357543:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1786177329.357545:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177329.357549:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177329.357551:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177329/real 1786177329] req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177329.357553:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177329.357558:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177329.357569:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.357575:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1786177329.357576:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177329.357580:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177329.357582:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000100:00000010:2.0:1786177329.357584:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000400:00000200:0.0:1786177329.357587:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000020:00000040:2.0:1786177329.357589:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000400:00000010:0.0:1786177329.357591:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 02000000:00000001:2.0:1786177329.357596:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1786177329.357597:0:2392:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1786177329.357601:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.357604:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:0.0:1786177329.357606:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177329.357606:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177329.357611:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177329.357619:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9000. 00000100:00000001:2.0:1786177329.357628:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1786177329.357632:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177329.357641:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177329.357644:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177329.357648:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.357656:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177329.357659:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177329.357667:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177329.357674:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177329.357675:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177329.357683:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.357724:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177329.357724:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177329.357733:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177329.357737:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177329.357738:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177329.357742:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1786177329.357743:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177329.357754:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177329.357761:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12003 00000400:00000200:2.0:1786177329.357772:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177329.357776:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:2.0:1786177329.357780:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:0.0:1786177329.357785:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1786177329.357787:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1786177329.357789:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177329.357793:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177329.357797:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177329.357798:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177329.357802:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177329.357805:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:0.0:1786177329.357809:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1786177329.357811:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00080000:0.0:1786177329.357816:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177329.357817:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177329.357821:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:2.0:1786177329.357823:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177329.357827:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000100:00000040:0.0:1786177329.357829:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177329.357833:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177329.357839:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177329.357848:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00100000:0.0:1786177329.357852:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235603840:192.168.202.132@tcp:250:kworker.0 00000020:00000001:2.0:1786177329.357854:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:0.0:1786177329.357862:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177329.357865:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177329.357867:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1786177329.357874:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177329.357875:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736a00 x1872930235603840/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177329.357885:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177329.357897:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177329.357898:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177329.357901:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:2.0:1786177329.357903:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 02000000:00000010:0.0:1786177329.357905:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177329.357911:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:2.0:1786177329.357912:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000020:00000001:0.0:1786177329.357915:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:2.0:1786177329.357918:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1786177329.357919:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177329.357926:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000001:2.0:1786177329.357926:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:0.0:1786177329.357931:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177329.357933:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177329.357936:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177329.357942:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1786177329.357942:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.357948:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000010:0.0:1786177329.357949:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736a00. 00000008:00100000:2.0:1786177329.357952:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1786177329.357955:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000008:00000001:2.0:1786177329.357956:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.357960:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177329.357960:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.357968:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1786177329.357969:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000020:00000040:2.0:1786177329.357976:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177329.357980:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1786177329.357981:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1786177329.357982:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.357988:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.357988:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177329.357990:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1786177329.357993:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1786177329.357994:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.357999:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1786177329.358001:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aabc00. 00000100:00000001:0.0:1786177329.358003:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1786177329.358007:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177329.358014:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177329.358017:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177329.358072:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177329.358080:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177329.358092:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358108:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177329.358113:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177329.358114:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177329.358117:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1786177329.358121:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12013) 02000000:00000001:0.0:1786177329.358122:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177329.358130:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177329.358136:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603968:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177329.358140:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177329.358142:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177329.358144:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177329.358145:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177329.358149:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177329.358151:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177329.358155:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177329.358158:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177329.358161:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177329.358164:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235603968, portal 10 00000100:00000001:0.0:1786177329.358166:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177329.358167:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469625472 : -110371239926144 : ffff9b9e31736680) 00000100:00000040:0.0:1786177329.358173:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358180:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177329.358182:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177329.358184:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177329.358186:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235603968, offset 0 00000400:00000200:0.0:1786177329.358189:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177329.358193:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177329.358196:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177329.358199:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177329.358201:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177329.358203:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177329.358206:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177329.358209:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426785:426785:256:4294967295] 192.168.202.132@tcp LPNI seq info [426747:426747:8:4294967295] 00000400:00000200:0.0:1786177329.358213:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177329.358220:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177329.358224:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177329.358226:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177329.358230:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177329.358233:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1786177329.358237:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177329.358240:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1786177329.358243:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177329.358245:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177329.358247:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177329.358250:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177329.358253:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177329.358255:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177329.358257:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177329.358260:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358267:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177329.358270:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358277:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.358278:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177329.358281:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177329.358284:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177329.358287:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177329.358289:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306676228 00000100:00000001:0.0:1786177329.358291:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306680065 00000100:00000001:0.0:1786177329.358293:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306676228 00000100:00000001:0.0:1786177329.358295:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306680065 00000100:00000001:0.0:1786177329.358297:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177329.358299:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.358301:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177329.358304:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177329/real 1786177329] req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177329.358311:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177329.358313:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177329.358314:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177329.358316:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177329.358318:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177329.358319:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177329.358322:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177329.358330:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358336:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177329.358338:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358339:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177329.358340:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177329.358344:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358350:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177329.358353:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358388:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177329.358391:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358392:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177329.358393:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358395:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177329.358397:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177329.358399:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177329.358403:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177329.358405:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177329.358409:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177329.358417:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235603968:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177329.358420:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177329.358424:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358430:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177329.358432:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177329.358434:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177329.358436:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177329.358438:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177329.358440:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604096:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177329.358443:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177329.358445:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177329.358446:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177329.358448:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177329.358451:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177329.358453:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177329.358456:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177329.358458:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177329.358460:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177329.358463:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235604096, portal 10 00000100:00000001:0.0:1786177329.358465:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177329.358466:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469623680 : -110371239927936 : ffff9b9e31735f80) 00000100:00000040:0.0:1786177329.358471:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358477:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177329.358480:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177329.358481:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177329.358484:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235604096, offset 0 00000400:00000200:0.0:1786177329.358487:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177329.358490:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177329.358493:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177329.358496:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177329.358498:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177329.358501:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177329.358503:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177329.358506:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426786:426786:256:4294967295] 192.168.202.132@tcp LPNI seq info [426748:426748:8:4294967295] 00000400:00000200:0.0:1786177329.358509:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177329.358514:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177329.358518:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177329.358521:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177329.358525:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177329.358528:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1786177329.358532:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177329.358534:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1786177329.358537:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177329.358539:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177329.358542:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177329.358545:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177329.358548:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177329.358549:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177329.358551:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177329.358555:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358562:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177329.358565:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358571:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.358573:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177329.358576:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177329.358578:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177329.358581:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177329.358583:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306676229 00000100:00000001:0.0:1786177329.358585:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306680065 00000100:00000001:0.0:1786177329.358588:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306676229 00000100:00000001:0.0:1786177329.358589:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306680065 00000100:00000001:0.0:1786177329.358591:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177329.358593:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.358595:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177329.358598:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177329/real 1786177329] req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177329.358605:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177329.358606:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177329.358607:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177329.358609:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177329.358611:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177329.358613:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177329.358616:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177329.358624:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358629:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177329.358631:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358632:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177329.358633:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177329.358637:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358643:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177329.358646:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358653:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177329.358655:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358657:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177329.358658:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358659:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177329.358661:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177329.358663:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177329.358666:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177329.358668:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177329.358671:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177329.358679:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604096:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177329.358682:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177329.358685:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358692:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177329.358693:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177329.358695:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177329.358696:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177329.358698:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177329.358701:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604224:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177329.358703:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177329.358705:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177329.358706:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177329.358708:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177329.358711:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177329.358712:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177329.358715:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177329.358718:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177329.358720:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177329.358723:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235604224, portal 4 00000100:00000001:0.0:1786177329.358724:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177329.358726:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469628160 : -110371239923456 : ffff9b9e31737100) 00000100:00000040:0.0:1786177329.358730:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358737:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177329.358739:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177329.358740:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177329.358742:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235604224, offset 0 00000400:00000200:0.0:1786177329.358746:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177329.358748:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177329.358751:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177329.358753:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177329.358756:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177329.358758:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177329.358760:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177329.358763:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426787:426787:256:4294967295] 192.168.202.132@tcp LPNI seq info [426749:426749:8:4294967295] 00000400:00000200:0.0:1786177329.358767:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177329.358773:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177329.358777:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177329.358779:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177329.358782:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177329.358785:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1786177329.358789:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177329.358791:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1786177329.358794:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177329.358796:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177329.358798:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177329.358802:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177329.358804:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177329.358806:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177329.358808:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177329.358811:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358818:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177329.358821:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358827:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.358829:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177329.358831:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177329.358833:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177329.358836:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177329.358838:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306676229 00000100:00000001:0.0:1786177329.358840:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306680065 00000100:00000001:0.0:1786177329.358842:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306676229 00000100:00000001:0.0:1786177329.358844:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306680065 00000100:00000001:0.0:1786177329.358846:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177329.358848:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.358849:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177329.358852:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177329/real 1786177329] req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177329.358859:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177329.358860:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177329.358861:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177329.358863:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177329.358864:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177329.358866:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177329.358869:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177329.358876:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358882:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177329.358884:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358886:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177329.358887:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177329.358890:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358896:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177329.358899:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358906:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177329.358908:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358909:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177329.358910:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358912:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177329.358913:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177329.358915:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177329.358917:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177329.358919:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177329.358923:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177329.358930:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604224:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177329.358933:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358935:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177329.358937:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736680 x1872930235603968/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358944:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177329.358946:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177329.358948:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177329.358950:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177329.358951:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177329.358953:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177329.358955:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177329.358956:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:0.0:1786177329.358959:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177329.358960:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177329.358962:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736680. 00000100:00000001:0.0:1786177329.358964:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177329.358965:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358967:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177329.358970:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735f80 x1872930235604096/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.358976:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177329.358977:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177329.358978:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177329.358980:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177329.358981:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177329.358982:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177329.358984:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177329.358985:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:0.0:1786177329.358987:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177329.358988:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177329.358989:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735f80. 00000100:00000001:0.0:1786177329.358991:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177329.358992:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.358993:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177329.358996:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737100 x1872930235604224/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.359002:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177329.359003:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177329.359004:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177329.359006:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177329.359007:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177329.359008:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177329.359010:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177329.359011:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:0.0:1786177329.359013:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177329.359014:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177329.359015:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737100. 00000100:00000001:0.0:1786177329.359017:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177329.359018:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.359020:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.359023:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177329.359024:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.359026:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177329.359027:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177329.359028:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177329.359031:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.359038:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177329.359039:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177329.359041:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177329.359042:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177329.359044:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177329.359047:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604352:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177329.359050:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177329.359051:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177329.359053:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177329.359054:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177329.359067:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177329.359070:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177329.359072:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177329.359074:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177329.359077:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177329.359080:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235604352, portal 4 00000100:00000001:0.0:1786177329.359081:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177329.359083:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469618304 : -110371239933312 : ffff9b9e31734a80) 00000100:00000040:0.0:1786177329.359088:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.359100:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177329.359102:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177329.359104:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177329.359106:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235604352, offset 0 00000400:00000200:0.0:1786177329.359110:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177329.359113:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177329.359115:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177329.359118:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177329.359120:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177329.359122:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177329.359124:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177329.359127:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426788:426788:256:4294967295] 192.168.202.132@tcp LPNI seq info [426750:426750:8:4294967295] 00000400:00000200:0.0:1786177329.359131:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177329.359137:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177329.359141:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177329.359144:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177329.359147:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177329.359150:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1786177329.359153:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177329.359156:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1786177329.359159:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177329.359161:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177329.359164:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177329.359167:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177329.359169:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177329.359171:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177329.359173:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177329.359176:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.359183:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177329.359186:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177354 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.359198:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.359199:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177329.359201:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177329.359204:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177329.359207:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177329.359209:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306676231 00000100:00000001:0.0:1786177329.359211:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306680065 00000100:00000001:0.0:1786177329.359213:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306676231 00000100:00000001:0.0:1786177329.359215:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306680065 00000100:00000001:0.0:1786177329.359217:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177329.359218:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.359220:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177329.359223:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177329/real 1786177329] req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177329.359230:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177329.359231:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177329.359233:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177329.359235:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177329.359236:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177329.359238:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177329.359241:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177329.359248:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.359255:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177329.359256:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.359258:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177329.359259:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177329.359262:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.359269:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177329.359273:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.359279:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177329.359281:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.359283:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177329.359284:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.359285:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177329.359286:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177329.359289:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177329.359291:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177329.359293:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177329.359297:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177329.359304:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604352:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177329.359307:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.359309:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177329.359311:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734a80 x1872930235604352/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177354 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177329.359317:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177329.359318:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177329.359320:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177329.359322:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177329.359323:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177329.359325:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177329.359327:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177329.359328:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:0.0:1786177329.359330:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177329.359331:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177329.359333:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734a80. 00000100:00000001:0.0:1786177329.359335:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177329.359336:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.359338:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177329.359340:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177329.359341:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177329.359343:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177329.359345:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177329.868451:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177329.868467:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177329.868471:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177329.868481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177329.868485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177329.868490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177329.868492:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177329.868496:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177329.868499:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177330.252423:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177330.252430:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177330.252431:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177330.252435:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1786177330.252436:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177330.252439:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177330.252440:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177330.252444:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177330.252447:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177330.252448:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177330.252451:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177330.252453:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177330.252454:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177330.252456:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177330.252460:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177330.252531:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177330.252539:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177330.252543:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177330.892440:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177330.892448:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177330.892451:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177330.892456:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177330.892460:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177330.892464:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177330.892466:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177330.892475:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177330.892478:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177331.276445:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177331.276451:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177331.276453:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177331.276456:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177331.276456:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177331.276464:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177331.276469:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177331.276470:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177331.276471:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177331.276474:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177331.276477:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177331.276479:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177331.276481:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1786177331.276482:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177331.276484:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177331.276485:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177331.276487:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177331.276492:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177331.276493:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177331.276503:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177331.276505:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177331.276591:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177331.276596:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177331.276599:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177331.916474:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177331.916482:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177331.916484:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177331.916490:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177331.916494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177331.916498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177331.916501:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177331.916505:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177331.916508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177332.300431:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177332.300436:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177332.300440:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177332.300441:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177332.300444:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177332.300446:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177332.300454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177332.300462:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177332.300464:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177332.300469:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177332.300474:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177332.300478:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177332.300480:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177332.300483:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177332.300493:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177332.300505:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177332.300508:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177332.300512:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177332.940479:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177332.940488:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177332.940490:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177332.940496:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177332.940500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177332.940503:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177332.940505:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177332.940508:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177332.940511:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177333.260465:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306680065, at 4306680128 00000100:00000040:1.0:1786177333.260465:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306680065, at 4306680128 00000100:00000010:1.0:1786177333.260482:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d56c0. 00000100:00000010:0.0:1786177333.260486:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5840. 00000100:00000001:3.0:1786177333.324488:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177333.324498:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177333.324498:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177333.324503:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177333.324507:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177333.324511:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177333.324516:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177333.324518:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177333.324521:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177333.324526:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177333.324532:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177333.324535:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177333.324540:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177333.324542:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177333.324546:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177333.324603:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177333.324609:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177333.324612:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177333.964495:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177333.964504:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177333.964506:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177333.964511:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177333.964516:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177333.964519:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177333.964523:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177333.964526:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177333.964530:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177334.348447:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177334.348453:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177334.348454:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177334.348459:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177334.348462:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177334.348463:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177334.348465:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177334.348468:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177334.348469:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1786177334.348470:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177334.348473:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1786177334.348473:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177334.348478:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177334.348483:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177334.348485:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177334.348487:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177334.348489:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177334.348492:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177334.348494:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177334.348504:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177334.348505:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177334.348507:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177334.348508:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177334.348509:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177334.348510:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177334.348511:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177334.348512:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177334.348606:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177334.348613:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177334.348617:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177334.476503:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177334.476518:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177334.476522:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177334.476527:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177334.476532:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177334.476541:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12008 00000400:00000200:2.0:1786177334.476552:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177334.476559:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177334.476563:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177334.476566:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.476569:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177334.476571:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.476573:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177334.476577:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177334.476578:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177334.476581:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177334.476583:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177334.476585:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177334.476586:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177334.476588:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.476590:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177334.476591:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.476592:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177334.476594:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177334.476596:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177334.476598:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177334.476600:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177334.476603:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177334.476605:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177334.476606:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177334.476608:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177334.476617:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734e00. 00000020:00000040:2.0:1786177334.476620:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177334.476624:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177334.476626:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177334.476630:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177334.476632:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177334.476636:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177334.476643:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9000. 00000100:00000001:2.0:1786177334.476650:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177334.476659:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177334.476669:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177334.476675:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177334.476680:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001f62212] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177334.476706:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177334.476710:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177334.476718:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177334.476720:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177334.476721:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177334.476725:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177334.476727:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12008 00000400:00000200:2.0:1786177334.476733:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177334.476736:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177334.476738:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177334.476741:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.476743:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177334.476746:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.476748:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177334.476751:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177334.476752:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177334.476754:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177334.476757:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177334.476760:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177334.476763:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177334.476766:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.476768:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177334.476771:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177334.476772:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177334.476774:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177334.476781:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:0.0:1786177334.476782:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177334.476787:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1786177334.476792:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177334.476799:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177334.476800:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177334.476806:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1786177334.476815:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1786177334.476817:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177334.476818:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177334.476826:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177334.476830:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177334.476832:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177334.476835:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177334.476837:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235604480:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177334.476844:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1786177334.476844:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:0.0:1786177334.476850:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1786177334.476852:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1786177334.476853:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1786177334.476861:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177334.476862:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1786177334.476870:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177334.476876:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735180. 02000000:00000010:0.0:1786177334.476877:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177334.476884:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177334.476884:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000200:0.0:1786177334.476897:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177334.476898:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177334.476905:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1786177334.476908:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000100:00000001:2.0:1786177334.476909:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177334.476913:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177334.476916:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1786177334.476917:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 02000000:00000010:2.0:1786177334.476921:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aabc00. 00000100:00000001:2.0:1786177334.476925:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1786177334.476927:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235604480, portal 25 00000100:00100000:2.0:1786177334.476931:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177334.476933:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177334.476937:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469619200 : -110371239932416 : ffff9b9e31734e00) 00000100:00000001:2.0:1786177334.476951:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1786177334.476952:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177334.476958:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177334.476970:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ba0f078] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.476971:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177334.476975:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177334.476977:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177334.476980:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235604480, offset 0 00000400:00000200:0.0:1786177334.476989:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177334.477016:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177334.477027:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177334.477030:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1786177334.477035:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00080000:2.0:1786177334.477040:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177334.477043:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177334.477046:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1786177334.477047:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1786177334.477057:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1786177334.477058:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00080000:2.0:1786177334.477063:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12008 00000400:00000200:0.0:1786177334.477064:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177334.477080:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177334.477081:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177334.477088:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:0.0:1786177334.477089:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426789:426789:256:4294967295] 192.168.202.132@tcp LPNI seq info [426751:426751:8:4294967295] 00000100:00080000:2.0:1786177334.477093:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177334.477101:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477105:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1786177334.477107:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:2.0:1786177334.477108:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477117:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177334.477118:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177334.477119:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177334.477121:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177334.477123:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177334.477128:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177334.477130:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177334.477139:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:0.0:1786177334.477142:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:2.0:1786177334.477153:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1786177334.477154:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000040:2.0:1786177334.477159:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477162:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000010:0.0:1786177334.477164:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000020:00000040:2.0:1786177334.477164:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177334.477176:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477179:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1786177334.477184:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 02000000:00000001:2.0:1786177334.477184:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177334.477189:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:0.0:1786177334.477193:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 02000000:00000001:2.0:1786177334.477194:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177334.477198:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177334.477200:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177334.477205:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1786177334.477210:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:2.0:1786177334.477211:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1786177334.477220:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000100:00000010:2.0:1786177334.477221:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000020:00000040:2.0:1786177334.477228:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177334.477235:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:0.0:1786177334.477236:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177334.477245:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 02000000:00000001:2.0:1786177334.477246:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177334.477249:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177334.477252:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177334.477254:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177334.477258:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000400:00000200:0.0:1786177334.477259:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1786177334.477262:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177334.477270:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1786177334.477274:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1786177334.477289:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1786177334.477290:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000100:00000001:2.0:1786177334.477291:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1786177334.477294:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000040:2.0:1786177334.477300:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.477303:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177334.477315:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177334.477316:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177334.477327:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177334.477345:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177334.477348:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1786177334.477349:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1786177334.477349:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177334.477353:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:0.0:1786177334.477354:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177334.477388:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12008 00000400:00000200:2.0:1786177334.477392:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177334.477395:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177334.477397:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177334.477400:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477402:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177334.477406:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477413:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177334.477420:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177334.477425:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177334.477428:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177334.477429:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177334.477431:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177334.477432:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177334.477435:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477436:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177334.477441:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477445:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177334.477447:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:0.0:1786177334.477448:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177334.477450:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477455:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.477456:0:2392:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1786177334.477457:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177334.477463:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177334.477466:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177334.477467:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177334.477469:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1786177334.477469:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177334.477472:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177334.477473:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177334.477476:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000020:00000040:2.0:1786177334.477482:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177334.477485:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177334.477487:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177334.477489:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177334.477490:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177334.477493:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177334.477498:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.477499:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306685185 02000000:00000010:2.0:1786177334.477501:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786177334.477506:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177334.477511:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:0.0:1786177334.477516:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5840. 00000100:00000001:2.0:1786177334.477516:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177334.477520:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177334.477524:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.477528:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177334.477531:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177334.477536:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177334.477539:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177334.477540:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1786177334.477541:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306685185 00000100:00080000:2.0:1786177334.477543:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177334.477546:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177334.477550:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12008 00000100:00000010:0.0:1786177334.477552:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5ea0. 00000400:00000200:2.0:1786177334.477557:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177334.477562:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000400:00000200:2.0:1786177334.477562:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177334.477569:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1786177334.477574:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1786177334.477575:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177334.477588:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177334.477588:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1786177334.477594:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1786177334.477594:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477599:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177334.477601:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177334.477602:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177334.477605:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177334.477608:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00100000:0.0:1786177334.477610:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177334/real 1786177334] req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1786177334.477611:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177334.477613:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177334.477626:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477635:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177334.477639:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477645:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177334.477652:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000400:00000001:0.0:1786177334.477654:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177334.477657:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1786177334.477657:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177334.477664:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1786177334.477665:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 02000000:00000001:2.0:1786177334.477667:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1786177334.477674:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 02000000:00000001:2.0:1786177334.477674:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1786177334.477683:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 02000000:00000001:2.0:1786177334.477683:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177334.477693:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177334.477695:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000008:00100000:2.0:1786177334.477696:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177334.477700:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177334.477701:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177334.477706:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000100:00000200:0.0:1786177334.477707:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1786177334.477709:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177334.477712:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177334.477714:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177334.477719:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177334.477721:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177334.477724:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177334.477731:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177334.477735:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177334.477749:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177334.477788:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:0.0:1786177334.477792:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177334.477792:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177334.477807:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.477810:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177334.477814:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.477816:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177334.477820:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177334.477831:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177334.477847:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.477848:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177334.477859:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12019) 00000100:00000040:0.0:1786177334.477860:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.477868:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177334.477870:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.477872:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177334.477872:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.477874:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177334.477876:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177334.477879:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177334.477884:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177334.477886:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177334.477889:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177334.477894:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235604480:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177334.477897:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.477898:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177334.477900:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734e00 x1872930235604480/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.477907:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177334.477909:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177334.477911:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177334.477914:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177334.477916:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177334.477917:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177334.477918:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177334.477920:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:0.0:1786177334.477924:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177334.477926:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177334.477927:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734e00. 00000100:00000001:0.0:1786177334.477931:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177334.477932:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.477934:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.477940:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177334.477942:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.477944:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177334.477945:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177334.477947:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177334.477951:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.477959:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177334.477961:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177334.477963:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177334.477965:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177334.477967:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177334.477969:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604608:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177334.477971:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177334.477973:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177334.477974:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177334.477975:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177334.477977:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177334.477978:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177334.477981:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177334.477983:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177334.477986:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177334.477988:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235604608, portal 10 00000100:00000001:0.0:1786177334.477989:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177334.477990:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620096 : -110371239931520 : ffff9b9e31735180) 00000100:00000040:0.0:1786177334.477994:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.477998:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177334.478000:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177334.478001:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177334.478002:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235604608, offset 0 00000400:00000200:0.0:1786177334.478005:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177334.478018:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177334.478021:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177334.478023:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177334.478025:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177334.478026:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177334.478028:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177334.478030:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426790:426790:256:4294967295] 192.168.202.132@tcp LPNI seq info [426752:426752:8:4294967295] 00000400:00000200:0.0:1786177334.478032:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177334.478037:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177334.478039:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177334.478041:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177334.478044:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177334.478046:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1786177334.478049:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177334.478051:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1786177334.478053:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177334.478054:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177334.478056:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177334.478058:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177334.478060:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177334.478061:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177334.478063:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177334.478065:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478070:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177334.478072:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478076:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.478077:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177334.478078:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177334.478081:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177334.478083:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177334.478084:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306681348 00000100:00000001:0.0:1786177334.478086:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306685185 00000100:00000001:0.0:1786177334.478088:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306681348 00000100:00000001:0.0:1786177334.478089:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306685185 00000100:00000001:0.0:1786177334.478090:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177334.478091:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.478093:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177334.478096:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177334/real 1786177334] req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177334.478103:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177334.478104:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177334.478106:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177334.478108:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177334.478110:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177334.478112:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177334.478115:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177334.478124:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478135:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177334.478137:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478139:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177334.478139:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177334.478142:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478146:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177334.478148:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478152:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177334.478153:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478154:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177334.478155:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478156:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177334.478157:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177334.478159:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177334.478160:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177334.478162:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177334.478164:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177334.478168:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604608:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177334.478170:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177334.478173:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478178:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177334.478179:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177334.478182:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177334.478183:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177334.478185:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177334.478188:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604736:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177334.478191:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177334.478194:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177334.478195:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177334.478197:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177334.478200:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177334.478202:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177334.478205:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177334.478208:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177334.478211:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177334.478214:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235604736, portal 10 00000100:00000001:0.0:1786177334.478216:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177334.478217:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469618304 : -110371239933312 : ffff9b9e31734a80) 00000100:00000040:0.0:1786177334.478222:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478226:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177334.478227:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177334.478229:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177334.478230:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235604736, offset 0 00000400:00000200:0.0:1786177334.478232:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177334.478235:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177334.478237:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177334.478239:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177334.478241:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177334.478243:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177334.478245:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177334.478248:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426791:426791:256:4294967295] 192.168.202.132@tcp LPNI seq info [426753:426753:8:4294967295] 00000400:00000200:0.0:1786177334.478252:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177334.478258:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177334.478263:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177334.478266:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177334.478269:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177334.478272:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1786177334.478276:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177334.478279:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1786177334.478282:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177334.478284:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177334.478287:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177334.478289:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177334.478291:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177334.478292:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177334.478293:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177334.478296:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478300:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177334.478302:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478306:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.478307:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177334.478309:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177334.478311:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177334.478312:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177334.478314:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306681348 00000100:00000001:0.0:1786177334.478315:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306685185 00000100:00000001:0.0:1786177334.478316:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306681348 00000100:00000001:0.0:1786177334.478318:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306685185 00000100:00000001:0.0:1786177334.478319:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177334.478320:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.478321:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177334.478323:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177334/real 1786177334] req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177334.478328:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177334.478329:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177334.478330:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177334.478331:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177334.478332:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177334.478333:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177334.478335:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177334.478339:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478343:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177334.478344:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478345:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177334.478346:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177334.478347:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478351:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177334.478353:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478376:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177334.478379:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478380:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177334.478381:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478383:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177334.478385:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177334.478387:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177334.478391:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177334.478393:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177334.478397:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177334.478405:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604736:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177334.478408:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177334.478412:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478418:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177334.478420:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177334.478422:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177334.478424:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177334.478426:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177334.478428:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604864:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177334.478432:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177334.478433:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177334.478435:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177334.478436:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177334.478439:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177334.478440:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177334.478443:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177334.478444:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177334.478447:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177334.478449:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235604864, portal 4 00000100:00000001:0.0:1786177334.478450:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177334.478451:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469628160 : -110371239923456 : ffff9b9e31737100) 00000100:00000040:0.0:1786177334.478455:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478459:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177334.478460:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177334.478461:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177334.478463:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235604864, offset 0 00000400:00000200:0.0:1786177334.478465:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177334.478468:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177334.478470:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177334.478472:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177334.478474:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177334.478475:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177334.478476:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177334.478478:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426792:426792:256:4294967295] 192.168.202.132@tcp LPNI seq info [426754:426754:8:4294967295] 00000400:00000200:0.0:1786177334.478481:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177334.478484:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177334.478487:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177334.478488:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177334.478491:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177334.478492:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1786177334.478495:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177334.478496:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1786177334.478498:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177334.478499:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177334.478501:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177334.478503:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177334.478505:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177334.478506:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177334.478508:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177334.478509:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478514:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177334.478516:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478519:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.478520:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177334.478521:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177334.478523:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177334.478525:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177334.478526:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306681349 00000100:00000001:0.0:1786177334.478528:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306685185 00000100:00000001:0.0:1786177334.478530:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306681349 00000100:00000001:0.0:1786177334.478531:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306685185 00000100:00000001:0.0:1786177334.478532:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177334.478533:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.478534:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177334.478536:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177334/real 1786177334] req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177334.478540:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177334.478541:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177334.478542:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177334.478544:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177334.478545:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177334.478546:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177334.478548:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177334.478552:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478556:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177334.478557:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478558:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177334.478559:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177334.478561:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478564:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177334.478566:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478569:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177334.478571:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478572:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177334.478573:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478574:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177334.478575:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177334.478576:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177334.478578:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177334.478579:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177334.478581:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177334.478585:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604864:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177334.478587:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177334.478589:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478593:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177334.478593:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177334.478594:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177334.478595:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177334.478597:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177334.478598:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604992:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177334.478600:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177334.478601:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177334.478602:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177334.478603:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177334.478605:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177334.478606:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177334.478607:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177334.478608:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177334.478610:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177334.478612:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235604992, portal 4 00000100:00000001:0.0:1786177334.478613:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177334.478614:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469623680 : -110371239927936 : ffff9b9e31735f80) 00000100:00000040:0.0:1786177334.478617:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478620:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177334.478622:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177334.478623:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177334.478624:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235604992, offset 0 00000400:00000200:0.0:1786177334.478626:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177334.478627:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177334.478629:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177334.478630:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177334.478631:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177334.478633:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177334.478634:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177334.478636:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426793:426793:256:4294967295] 192.168.202.132@tcp LPNI seq info [426755:426755:8:4294967295] 00000400:00000200:0.0:1786177334.478638:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177334.478642:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177334.478644:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177334.478645:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177334.478647:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177334.478649:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1786177334.478651:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177334.478653:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1786177334.478655:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177334.478656:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177334.478657:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177334.478659:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177334.478661:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177334.478662:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177334.478663:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177334.478664:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478672:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177334.478674:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177359 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478677:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.478678:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177334.478691:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177334.478694:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177334.478697:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177334.478699:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306681351 00000100:00000001:0.0:1786177334.478700:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306685185 00000100:00000001:0.0:1786177334.478702:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306681351 00000100:00000001:0.0:1786177334.478704:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306685185 00000100:00000001:0.0:1786177334.478706:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177334.478707:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.478709:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177334.478713:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177334/real 1786177334] req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177334.478720:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177334.478721:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177334.478723:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177334.478725:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177334.478727:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177334.478729:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177334.478732:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177334.478740:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478744:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177334.478746:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478747:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177334.478748:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177334.478750:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478753:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177334.478756:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478759:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177334.478761:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478761:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177334.478762:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478763:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177334.478764:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177334.478765:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177334.478767:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177334.478768:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177334.478770:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177334.478774:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235604992:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177334.478776:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478777:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177334.478778:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735180 x1872930235604608/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478782:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177334.478783:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177334.478785:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177334.478786:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177334.478787:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177334.478788:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177334.478789:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177334.478790:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:0.0:1786177334.478792:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177334.478793:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177334.478794:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735180. 00000100:00000001:0.0:1786177334.478795:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177334.478796:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478797:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177334.478799:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734a80 x1872930235604736/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478802:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177334.478803:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177334.478804:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177334.478805:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177334.478805:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177334.478806:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177334.478807:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177334.478808:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:0.0:1786177334.478809:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177334.478809:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177334.478810:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734a80. 00000100:00000001:0.0:1786177334.478811:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177334.478812:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478812:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177334.478814:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737100 x1872930235604864/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478817:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177334.478818:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177334.478818:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177334.478819:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177334.478820:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177334.478821:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177334.478821:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177334.478822:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:0.0:1786177334.478823:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177334.478824:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177334.478825:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737100. 00000100:00000001:0.0:1786177334.478826:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177334.478827:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478828:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177334.478830:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735f80 x1872930235604992/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177359 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177334.478833:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177334.478834:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177334.478834:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177334.478835:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177334.478836:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177334.478837:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177334.478837:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177334.478838:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:0.0:1786177334.478839:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177334.478840:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177334.478840:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735f80. 00000100:00000001:0.0:1786177334.478841:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177334.478842:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478843:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177334.478845:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177334.478846:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177334.478847:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177334.478848:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177334.988480:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177334.988488:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177334.988490:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177334.988496:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177334.988500:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177334.988504:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177334.988507:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177334.988510:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177334.988512:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177335.372469:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177335.372469:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177335.372477:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177335.372479:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177335.372485:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177335.372485:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177335.372489:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177335.372494:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177335.372497:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177335.372498:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177335.372501:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177335.372502:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177335.372508:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177335.372508:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177335.372512:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177335.372611:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177335.372618:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177335.372623:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177336.012539:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177336.012553:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177336.012556:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177336.012564:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177336.012570:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177336.012576:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177336.012580:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177336.012584:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177336.012588:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177336.396520:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177336.396533:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177336.396535:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177336.396544:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177336.396549:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177336.396555:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177336.396558:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177336.396561:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177336.396565:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177336.397139:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177336.397203:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177336.397212:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177336.397214:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177336.397218:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177336.397219:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177336.397332:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177336.397336:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177336.397341:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177337.036647:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177337.036654:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177337.036656:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177337.036661:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177337.036664:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177337.036668:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177337.036670:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177337.036673:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177337.036676:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177337.420472:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177337.420480:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177337.420482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177337.420488:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1786177337.420488:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177337.420495:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177337.420495:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177337.420500:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177337.420503:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177337.420505:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177337.420506:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177337.420510:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177337.420512:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177337.420516:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177337.420516:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177337.420591:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177337.420595:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177337.420598:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177338.060477:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177338.060485:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177338.060488:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177338.060494:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177338.060499:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177338.060502:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177338.060504:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177338.060506:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177338.060508:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177338.380569:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306685185, at 4306685248 00000100:00000010:0.0:1786177338.380584:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5840. 00000100:00000040:1.0:1786177338.381507:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306685185, at 4306685249 00000100:00000010:1.0:1786177338.381515:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5ea0. 00000100:00000001:3.0:1786177338.444463:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177338.444467:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177338.444470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177338.444472:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177338.444475:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177338.444480:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1786177338.444480:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177338.444484:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177338.444485:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177338.444489:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177338.444492:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177338.444492:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177338.444496:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177338.444496:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177338.444501:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177338.444544:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177338.444547:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177338.444550:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177339.084475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177339.084483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177339.084487:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177339.084492:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177339.084496:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177339.084500:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177339.084503:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177339.084506:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177339.084509:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177339.468446:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177339.468449:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177339.468452:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177339.468454:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177339.468457:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177339.468458:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177339.468459:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177339.468463:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177339.468464:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177339.468464:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177339.468469:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177339.468472:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177339.468474:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177339.468477:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177339.468479:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177339.468515:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177339.468518:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177339.468520:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177339.596483:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177339.596493:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177339.596496:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177339.596500:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177339.596516:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177339.596525:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12014 00000400:00000200:3.0:1786177339.596535:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177339.596541:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177339.596544:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177339.596548:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177339.596551:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177339.596553:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177339.596556:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177339.596561:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177339.596563:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177339.596566:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177339.596569:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177339.596572:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177339.596575:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177339.596578:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177339.596580:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177339.596582:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177339.596584:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177339.596586:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177339.596589:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177339.596591:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177339.596594:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177339.596597:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177339.596600:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177339.596602:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177339.596604:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177339.596614:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786177339.596617:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177339.596621:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177339.596623:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177339.596626:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177339.596629:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177339.596633:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177339.596641:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786177339.596647:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177339.596657:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177339.596667:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177339.596672:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177339.596677:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177339.596703:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177339.596707:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177339.596710:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177339.596711:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177339.596713:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177339.596715:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177339.596718:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12014 00000400:00000200:3.0:1786177339.596723:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177339.596726:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:0.0:1786177339.596728:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1786177339.596729:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1786177339.596732:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1786177339.596735:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177339.596736:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177339.596740:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177339.596744:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177339.596745:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177339.596751:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177339.596756:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1786177339.596758:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1786177339.596759:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000100:00000001:0.0:1786177339.596761:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177339.596764:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:0.0:1786177339.596765:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177339.596769:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1786177339.596769:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177339.596775:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 02000000:00000001:0.0:1786177339.596776:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177339.596781:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235605120:192.168.202.132@tcp:250:kworker.0 00000020:00000001:3.0:1786177339.596781:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:0.0:1786177339.596787:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1786177339.596789:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:0.0:1786177339.596791:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177339.596796:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177339.596796:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1786177339.596800:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1786177339.596801:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000010:0.0:1786177339.596805:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:3.0:1786177339.596805:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:0.0:1786177339.596810:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1786177339.596811:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000400:00000200:0.0:1786177339.596816:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:3.0:1786177339.596818:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000010:0.0:1786177339.596820:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000100:00000001:3.0:1786177339.596822:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1786177339.596826:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 02000000:00000001:3.0:1786177339.596827:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1786177339.596832:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235605120, portal 25 02000000:00000001:3.0:1786177339.596834:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177339.596835:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177339.596838:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280816256 : -110371428735360 : ffff9b9e26326680) 02000000:00000001:3.0:1786177339.596839:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177339.596843:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177339.596846:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177339.596850:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177339.596851:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177339.596853:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177339.596860:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327100. 00000100:00000001:0.0:1786177339.596861:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177339.596865:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000020:00000040:3.0:1786177339.596867:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000400:00000010:0.0:1786177339.596869:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177339.596872:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235605120, offset 0 02000000:00000001:3.0:1786177339.596872:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177339.596875:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177339.596878:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1786177339.596879:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 02000000:00000001:3.0:1786177339.596881:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177339.596884:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177339.596888:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:3.0:1786177339.596892:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000400:00000200:0.0:1786177339.596894:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177339.596900:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1786177339.596900:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177339.596907:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00100000:3.0:1786177339.596911:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1786177339.596912:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177339.596916:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177339.596922:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426794:426794:256:4294967295] 192.168.202.132@tcp LPNI seq info [426756:426756:8:4294967295] 00000100:00000001:3.0:1786177339.596933:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1786177339.596935:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177339.596938:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177339.596946:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c8911fc] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1786177339.596948:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177339.596968:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177339.596976:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000100:00000001:3.0:1786177339.596977:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177339.596982:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1786177339.596985:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786177339.596989:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:0.0:1786177339.596990:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1786177339.596992:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177339.596996:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:0.1:1786177339.596996:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177339.597001:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1786177339.597003:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:0.0:1786177339.597006:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00080000:3.0:1786177339.597008:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12014 00000800:00000010:0.0:1786177339.597010:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177339.597016:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1786177339.597016:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177339.597025:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:0.0:1786177339.597026:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1786177339.597032:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1786177339.597033:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177339.597037:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000040:3.0:1786177339.597040:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177339.597042:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786177339.597043:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1786177339.597047:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177339.597048:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177339.597054:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177339.597059:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1786177339.597062:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1786177339.597062:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000100:00000040:0.0:1786177339.597066:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1786177339.597067:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177339.597071:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177339.597076:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000001:0.0:1786177339.597077:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.597081:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1786177339.597082:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000001:0.0:1786177339.597086:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177339.597088:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:0.0:1786177339.597092:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177339.597094:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1786177339.597096:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306690304 00000100:00000040:3.0:1786177339.597099:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000010:0.0:1786177339.597102:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5840. 00000100:00000001:3.0:1786177339.597103:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177339.597107:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00000001:0.0:1786177339.597110:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1786177339.597111:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177339.597116:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177339.597118:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1786177339.597119:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306690304 02000000:00000001:3.0:1786177339.597122:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:0.0:1786177339.597123:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5720. 00000100:00000001:0.0:1786177339.597128:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1786177339.597129:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.597131:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1786177339.597133:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177339.597134:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.597137:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000008:00100000:3.0:1786177339.597137:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177339.597142:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177339.597143:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177339/real 1786177339] req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177339.597147:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177339.597154:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324380. 00000400:00000001:0.0:1786177339.597155:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177339.597157:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:3.0:1786177339.597160:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000010:0.0:1786177339.597161:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177339.597164:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177339.597168:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 02000000:00000001:3.0:1786177339.597168:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177339.597171:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.597172:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1786177339.597176:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:0.0:1786177339.597177:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177339.597179:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177339.597183:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177339.597191:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000200:0.0:1786177339.597193:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177339.597197:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177339.597204:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177339.597210:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177339.597212:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.597215:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177339.597217:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177339.597220:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1786177339.597223:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177339.597226:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.597233:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1786177339.597236:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000095147275] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786177339.597240:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.597263:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177339.597264:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.597265:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.597269:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177339.597270:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177339.597270:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1786177339.597275:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1786177339.597278:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786177339.597279:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177339.597284:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1786177339.597284:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177339.597288:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1786177339.597289:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177339.597294:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1786177339.597295:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:0.0:1786177339.597298:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1786177339.597302:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12014 00000400:00000200:3.0:1786177339.597310:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:0.0:1786177339.597314:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235605120:192.168.202.132@tcp:250:kworker.0 00000400:00000200:3.0:1786177339.597316:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:0.0:1786177339.597319:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1786177339.597321:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1786177339.597322:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177339.597327:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326680 x1872930235605120/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177339.597329:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177339.597334:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177339.597339:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177339.597346:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1786177339.597346:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:0.0:1786177339.597348:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177339.597351:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 00000020:00000001:3.0:1786177339.597351:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177339.597354:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 02000000:00000001:0.0:1786177339.597379:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177339.597381:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177339.597382:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177339.597384:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177339.597386:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 02000000:00000001:0.0:1786177339.597389:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177339.597392:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:3.0:1786177339.597393:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000010:0.0:1786177339.597394:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326680. 00000020:00000040:3.0:1786177339.597397:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000100:00000001:0.0:1786177339.597398:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177339.597400:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.597404:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1786177339.597404:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177339.597409:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:0.0:1786177339.597411:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177339.597413:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177339.597416:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177339.597417:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177339.597419:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177339.597421:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177339.597422:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177339.597426:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1786177339.597428:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177339.597434:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177339.597439:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1786177339.597440:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:0.0:1786177339.597441:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177339.597444:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1786177339.597446:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:0.0:1786177339.597447:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177339.597450:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177339.597453:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177339.597454:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235605248:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177339.597458:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1786177339.597458:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177339.597461:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177339.597463:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177339.597463:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177339.597464:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177339.597468:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:3.0:1786177339.597468:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177339.597471:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177339.597473:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:0.0:1786177339.597476:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000008:00100000:3.0:1786177339.597477:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000400:00000010:0.0:1786177339.597479:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177339.597484:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000008:00000001:3.0:1786177339.597485:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1786177339.597488:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235605248, portal 10 00000100:00000001:3.0:1786177339.597489:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.597492:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177339.597494:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818944 : -110371428732672 : ffff9b9e26327100) 00000100:00000010:3.0:1786177339.597496:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326680. 00000100:00000040:0.0:1786177339.597500:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177339.597501:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177339.597507:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1786177339.597509:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1786177339.597510:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177339.597512:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177339.597515:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1786177339.597516:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 02000000:00000001:3.0:1786177339.597519:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:0.0:1786177339.597520:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235605248, offset 0 00000400:00000200:0.0:1786177339.597524:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 02000000:00000001:3.0:1786177339.597524:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177339.597529:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:3.0:1786177339.597531:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000400:00000200:0.0:1786177339.597533:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177339.597538:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1786177339.597538:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177339.597542:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177339.597544:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177339.597548:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00100000:3.0:1786177339.597549:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1786177339.597550:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426795:426795:256:4294967295] 192.168.202.132@tcp LPNI seq info [426757:426757:8:4294967295] 00000400:00000200:0.0:1786177339.597563:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:3.0:1786177339.597569:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177339.597574:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177339.597577:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000040:3.0:1786177339.597582:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdbcce8e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1786177339.597588:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177339.597596:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000100:00000001:3.0:1786177339.597599:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1786177339.597604:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00080000:3.0:1786177339.597605:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1786177339.597609:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00080000:3.0:1786177339.597611:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000200:0.1:1786177339.597616:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177339.597617:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:0.1:1786177339.597619:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00080000:3.0:1786177339.597621:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1786177339.597624:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:3.0:1786177339.597626:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000800:00000010:0.0:1786177339.597627:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000100:00080000:3.0:1786177339.597631:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12014 00000400:00000200:0.0:1786177339.597633:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1786177339.597638:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177339.597643:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1786177339.597647:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000200:0.0:1786177339.597649:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000100:00080000:3.0:1786177339.597652:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1786177339.597654:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000040:3.0:1786177339.597657:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177339.597659:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1786177339.597661:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177339.597666:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:0.0:1786177339.597667:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177339.597670:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177339.597677:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000100:00000001:0.0:1786177339.597680:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1786177339.597680:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177339.597684:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:0.0:1786177339.597685:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1786177339.597689:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:0.0:1786177339.597694:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177339.597694:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:0.0:1786177339.597697:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1786177339.597699:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:0.0:1786177339.597701:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177339.597705:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000040:0.0:1786177339.597706:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000100:00000001:3.0:1786177339.597711:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1786177339.597712:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177339.597715:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306686468 00000100:00000040:3.0:1786177339.597717:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177339.597718:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306690304 00000100:00000001:3.0:1786177339.597721:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177339.597722:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306686468 00000020:00000040:3.0:1786177339.597726:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:0.0:1786177339.597727:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306690304 00000100:00000001:0.0:1786177339.597732:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177339.597735:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177339.597735:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177339.597739:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1786177339.597743:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177339.597745:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177339/real 1786177339] req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177339.597747:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177339.597751:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:0.0:1786177339.597752:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177339.597755:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177339.597757:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 02000000:00000001:3.0:1786177339.597757:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177339.597762:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000100:00000001:3.0:1786177339.597763:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:0.0:1786177339.597764:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177339.597767:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000008:00100000:3.0:1786177339.597768:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000200:0.0:1786177339.597769:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1786177339.597776:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1786177339.597781:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177339.597781:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177339.597788:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327480. 00000020:00000040:3.0:1786177339.597793:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000001:0.0:1786177339.597798:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1786177339.597799:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1786177339.597800:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.597804:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1786177339.597804:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.597805:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177339.597810:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1786177339.597811:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1786177339.597814:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177339.597818:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.597819:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177339.597825:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1786177339.597826:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:0.0:1786177339.597833:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1786177339.597833:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.597838:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.597840:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177339.597841:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.597843:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00100000:3.0:1786177339.597845:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177339.597846:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177339.597850:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177339.597855:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177339.597857:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177339.597863:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177339.597870:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1786177339.597871:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235605248:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177339.597877:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1786177339.597877:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177339.597880:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.597886:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177339.597888:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177339.597889:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f76ab1c9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786177339.597890:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177339.597892:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177339.597895:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177339.597900:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235605376:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177339.597906:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177339.597908:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177339.597910:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177339.597914:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1786177339.597916:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177339.597917:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177339.597921:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177339.597923:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177339.597925:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000100:00000040:3.0:1786177339.597925:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12024) 00000400:00000010:0.0:1786177339.597930:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177339.597933:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235605376, portal 10 00000100:00000001:0.0:1786177339.597934:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177339.597935:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280807296 : -110371428744320 : ffff9b9e26324380) 00000100:00000040:0.0:1786177339.597940:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.597944:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177339.597946:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177339.597948:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177339.597949:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235605376, offset 0 00000400:00000200:0.0:1786177339.597951:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177339.597954:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177339.597970:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177339.597972:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177339.597974:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177339.597976:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177339.597977:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177339.597979:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426796:426796:256:4294967295] 192.168.202.132@tcp LPNI seq info [426758:426758:8:4294967295] 00000400:00000200:0.0:1786177339.597982:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177339.597986:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177339.597989:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177339.597991:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177339.597994:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177339.597996:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1786177339.597998:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177339.598000:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1786177339.598003:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177339.598004:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177339.598006:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177339.598008:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177339.598009:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177339.598011:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177339.598012:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177339.598014:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598019:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177339.598021:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598025:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.598026:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177339.598027:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177339.598029:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177339.598031:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177339.598032:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306686468 00000100:00000001:0.0:1786177339.598034:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306690304 00000100:00000001:0.0:1786177339.598035:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306686468 00000100:00000001:0.0:1786177339.598036:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306690304 00000100:00000001:0.0:1786177339.598037:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177339.598038:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.598040:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177339.598042:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177339/real 1786177339] req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177339.598047:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177339.598047:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177339.598049:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177339.598050:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177339.598051:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177339.598052:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177339.598053:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177339.598058:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598062:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177339.598062:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598063:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177339.598064:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177339.598066:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598069:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177339.598071:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598075:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177339.598076:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598077:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177339.598077:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598078:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177339.598080:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177339.598081:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177339.598082:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177339.598083:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177339.598086:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177339.598091:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235605376:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177339.598093:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598094:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177339.598095:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327100 x1872930235605248/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598099:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177339.598100:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177339.598101:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177339.598102:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177339.598103:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177339.598103:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177339.598104:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177339.598105:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa800. 02000000:00000001:0.0:1786177339.598107:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177339.598108:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177339.598108:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327100. 00000100:00000001:0.0:1786177339.598110:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177339.598111:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598112:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177339.598113:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324380 x1872930235605376/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598117:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177339.598118:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177339.598118:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177339.598119:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177339.598120:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177339.598121:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177339.598122:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177339.598122:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9000. 02000000:00000001:0.0:1786177339.598123:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177339.598124:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177339.598125:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324380. 00000100:00000001:0.0:1786177339.598127:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177339.598127:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598129:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598131:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177339.598132:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.598133:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177339.598134:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177339.598135:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177339.598137:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598142:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177339.598142:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177339.598144:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177339.598145:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177339.598146:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177339.598147:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235605504:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177339.598149:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177339.598150:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177339.598151:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177339.598152:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177339.598153:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177339.598154:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177339.598156:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177339.598157:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177339.598159:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177339.598161:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235605504, portal 4 00000100:00000001:0.0:1786177339.598163:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177339.598164:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280816256 : -110371428735360 : ffff9b9e26326680) 00000100:00000040:0.0:1786177339.598167:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598173:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177339.598175:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177339.598176:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177339.598178:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235605504, offset 0 00000400:00000200:0.0:1786177339.598180:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177339.598182:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177339.598184:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177339.598186:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177339.598188:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177339.598189:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177339.598190:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177339.598192:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426797:426797:256:4294967295] 192.168.202.132@tcp LPNI seq info [426759:426759:8:4294967295] 00000400:00000200:0.0:1786177339.598195:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177339.598198:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177339.598200:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177339.598202:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177339.598204:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177339.598206:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1786177339.598208:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177339.598210:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1786177339.598212:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177339.598213:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177339.598215:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177339.598217:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177339.598219:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177339.598220:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177339.598221:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177339.598223:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598228:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177339.598229:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598233:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.598234:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177339.598236:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177339.598238:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177339.598240:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177339.598241:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306686468 00000100:00000001:0.0:1786177339.598242:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306690304 00000100:00000001:0.0:1786177339.598244:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306686468 00000100:00000001:0.0:1786177339.598245:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306690304 00000100:00000001:0.0:1786177339.598246:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177339.598248:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.598249:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177339.598251:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177339/real 1786177339] req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177339.598255:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177339.598255:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177339.598256:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177339.598257:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177339.598258:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177339.598259:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177339.598261:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177339.598265:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598269:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177339.598270:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598271:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177339.598271:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177339.598273:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598277:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177339.598279:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598283:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177339.598284:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598285:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177339.598286:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598287:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177339.598288:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177339.598289:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177339.598291:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177339.598292:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177339.598294:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177339.598298:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235605504:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177339.598300:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177339.598301:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598305:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177339.598305:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177339.598306:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177339.598307:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177339.598308:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177339.598310:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235605632:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177339.598311:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177339.598312:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177339.598313:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177339.598314:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177339.598316:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177339.598317:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177339.598319:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177339.598321:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177339.598322:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177339.598324:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235605632, portal 4 00000100:00000001:0.0:1786177339.598325:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177339.598326:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280819840 : -110371428731776 : ffff9b9e26327480) 00000100:00000040:0.0:1786177339.598330:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598333:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177339.598334:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177339.598336:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177339.598337:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235605632, offset 0 00000400:00000200:0.0:1786177339.598338:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177339.598340:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177339.598342:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177339.598343:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177339.598345:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177339.598346:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177339.598347:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177339.598349:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426798:426798:256:4294967295] 192.168.202.132@tcp LPNI seq info [426760:426760:8:4294967295] 00000400:00000200:0.0:1786177339.598351:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177339.598354:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177339.598371:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177339.598373:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177339.598375:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177339.598377:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1786177339.598379:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177339.598381:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1786177339.598383:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177339.598384:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177339.598386:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177339.598388:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177339.598390:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177339.598391:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177339.598393:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177339.598395:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598399:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177339.598401:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177364 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598404:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.598405:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177339.598407:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177339.598408:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177339.598410:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177339.598411:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306686471 00000100:00000001:0.0:1786177339.598412:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306690304 00000100:00000001:0.0:1786177339.598413:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306686471 00000100:00000001:0.0:1786177339.598414:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306690304 00000100:00000001:0.0:1786177339.598416:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177339.598417:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.598418:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177339.598420:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177339/real 1786177339] req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177339.598424:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177339.598425:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177339.598425:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177339.598426:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177339.598427:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177339.598428:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177339.598430:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177339.598435:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598442:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177339.598443:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598444:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177339.598445:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177339.598447:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598450:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177339.598452:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598455:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177339.598456:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598457:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177339.598458:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598459:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177339.598460:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177339.598461:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177339.598463:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177339.598465:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177339.598467:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177339.598471:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235605632:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177339.598473:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598474:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177339.598475:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326680 x1872930235605504/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598478:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177339.598479:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177339.598481:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177339.598482:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177339.598483:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177339.598484:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177339.598485:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177339.598486:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9400. 02000000:00000001:0.0:1786177339.598487:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177339.598488:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177339.598489:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326680. 00000100:00000001:0.0:1786177339.598490:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177339.598491:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598492:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177339.598494:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327480 x1872930235605632/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177364 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177339.598497:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177339.598498:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177339.598498:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177339.598499:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177339.598500:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177339.598500:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177339.598501:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177339.598502:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa000. 02000000:00000001:0.0:1786177339.598503:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177339.598503:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177339.598504:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327480. 00000100:00000001:0.0:1786177339.598505:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177339.598506:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598507:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177339.598508:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177339.598509:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177339.598510:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177339.598511:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177340.108437:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177340.108445:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177340.108447:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177340.108453:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177340.108457:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177340.108460:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177340.108463:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177340.108466:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177340.108468:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177340.492468:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177340.492468:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177340.492476:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177340.492478:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177340.492480:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177340.492484:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177340.492487:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177340.492488:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177340.492492:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177340.492492:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177340.492496:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177340.492497:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1786177340.492499:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177340.492504:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177340.492506:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177340.492514:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177340.492516:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177340.492519:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177340.492521:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177340.492525:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177340.492526:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177340.492568:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177340.492572:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177340.492576:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177341.132651:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177341.132660:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177341.132662:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177341.132668:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177341.132671:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177341.132674:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177341.132676:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177341.132679:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177341.132682:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177341.516475:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177341.516482:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177341.516489:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177341.516490:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177341.516493:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177341.516495:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177341.516513:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177341.516519:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177341.516521:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177341.516526:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177341.516531:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177341.516535:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177341.516538:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177341.516540:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177341.516555:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177341.516564:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177341.516569:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177341.516572:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177342.156489:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177342.156496:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177342.156498:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177342.156504:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177342.156508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177342.156511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177342.156514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177342.156517:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177342.156520:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177342.540473:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177342.540480:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177342.540482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177342.540485:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177342.540494:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177342.540498:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177342.540500:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177342.540503:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177342.540506:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177342.540618:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177342.540621:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177342.540625:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177342.540626:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177342.540629:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177342.540631:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177342.540695:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177342.540698:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177342.540701:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177343.180429:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177343.180436:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177343.180439:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177343.180444:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177343.180448:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177343.180451:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177343.180455:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177343.180458:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177343.180471:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177343.500467:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306690304, at 4306690368 00000100:00000040:1.0:1786177343.500470:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306690304, at 4306690368 00000100:00000010:1.0:1786177343.500482:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5720. 00000100:00000010:0.0:1786177343.500486:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5840. 00000100:00000001:1.0:1786177343.564477:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177343.564486:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177343.564488:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177343.564494:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177343.564495:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177343.564498:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177343.564501:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1786177343.564504:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177343.564505:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177343.564510:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177343.564514:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177343.564516:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177343.564518:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177343.564522:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177343.564522:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177343.564527:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177343.564528:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177343.564531:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177343.564532:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177343.564536:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177343.564539:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177343.564540:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177343.564542:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177343.564544:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177343.564545:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177343.564548:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177343.564550:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177343.564668:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177343.564672:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177343.564674:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177344.204408:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177344.204414:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177344.204416:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177344.204423:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177344.204427:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177344.204432:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177344.204434:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177344.204437:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177344.204441:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.588435:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177344.588437:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177344.588442:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.588443:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177344.588447:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177344.588450:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177344.588452:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177344.588453:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177344.588455:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177344.588457:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.588458:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177344.588462:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177344.588465:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177344.588467:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177344.588471:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177344.588504:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177344.588507:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177344.588510:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177344.716481:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177344.716491:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177344.716494:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177344.716497:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177344.716500:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177344.716505:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12019 00000400:00000200:1.0:1786177344.716511:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177344.716515:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177344.716518:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177344.716520:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.716524:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177344.716526:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.716529:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177344.716532:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177344.716535:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177344.716537:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177344.716540:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177344.716543:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177344.716545:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177344.716548:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.716549:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177344.716551:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.716553:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177344.716556:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177344.716559:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177344.716562:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177344.716564:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177344.716567:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177344.716568:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.716570:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177344.716572:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177344.716579:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8380. 00000020:00000040:1.0:1786177344.716582:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177344.716585:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177344.716586:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.716588:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177344.716590:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177344.716593:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177344.716598:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad400. 00000100:00000001:1.0:1786177344.716604:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177344.716614:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177344.716623:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177344.716627:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177344.716632:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002fbc72ce] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177344.716651:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177344.716654:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177344.716657:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177344.716658:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177344.716660:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177344.716663:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177344.716666:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12019 00000100:00000001:0.0:1786177344.716671:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1786177344.716672:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177344.716676:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1786177344.716678:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:0.0:1786177344.716679:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1786177344.716681:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177344.716687:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177344.716688:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.716694:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177344.716697:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177344.716699:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177344.716700:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177344.716701:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177344.716704:0:191280:0:(genops.c:813:class_conn2export()) Process entered 02000000:00000001:0.0:1786177344.716706:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1786177344.716706:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 02000000:00000001:0.0:1786177344.716710:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1786177344.716710:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177344.716712:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 02000000:00000001:0.0:1786177344.716713:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177344.716715:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00100000:0.0:1786177344.716718:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235605760:192.168.202.132@tcp:250:kworker.0 00000020:00000001:1.0:1786177344.716719:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:0.0:1786177344.716724:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1786177344.716725:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.716727:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1786177344.716728:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1786177344.716729:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 02000000:00000001:0.0:1786177344.716731:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.716731:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:0.0:1786177344.716734:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:1.0:1786177344.716735:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:1.0:1786177344.716737:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000010:0.0:1786177344.716739:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:1.0:1786177344.716739:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177344.716743:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177344.716744:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177344.716747:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177344.716750:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177344.716753:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177344.716754:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177344.716758:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1786177344.716760:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000008:00000001:1.0:1786177344.716761:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.716764:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177344.716768:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9880. 00000400:00000010:0.0:1786177344.716769:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000020:00000040:1.0:1786177344.716770:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177344.716773:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177344.716775:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.716776:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177344.716777:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:0.0:1786177344.716778:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235605760, portal 25 02000000:00000001:1.0:1786177344.716779:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.716784:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 02000000:00000010:1.0:1786177344.716786:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae000. 00000100:00000001:0.0:1786177344.716787:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982976 : -110371221568640 : ffff9b9e328b8380) 00000100:00000001:1.0:1786177344.716790:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177344.716797:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1786177344.716798:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177344.716814:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177344.716817:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.716818:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1786177344.716822:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b2b2226e] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1786177344.716823:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177344.716834:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177344.716838:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235605760, offset 0 00000100:00000001:1.0:1786177344.716845:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177344.716848:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:1.0:1786177344.716850:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177344.716855:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177344.716856:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1786177344.716859:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177344.716859:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177344.716864:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177344.716868:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12019 00000400:00000200:0.0:1786177344.716871:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177344.716877:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177344.716882:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177344.716886:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177344.716888:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177344.716892:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426799:426799:256:4294967295] 192.168.202.132@tcp LPNI seq info [426761:426761:8:4294967295] 00000400:00000200:0.0:1786177344.716896:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177344.716905:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177344.716921:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000400:00000200:1.0:1786177344.716922:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177344.716928:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000800:00000010:0.0:1786177344.716930:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000100:00080000:1.0:1786177344.716930:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177344.716935:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.716937:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177344.716939:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.716943:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:0.0:1786177344.716944:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000020:00000001:1.0:1786177344.716946:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177344.716949:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177344.716950:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:0.0:1786177344.716951:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000020:00000040:1.0:1786177344.716952:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177344.716955:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177344.716958:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177344.716962:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000800:00000200:0.1:1786177344.716962:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177344.716966:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177344.716968:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000800:00000200:0.1:1786177344.716969:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000100:00000001:1.0:1786177344.716970:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177344.716976:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:1.0:1786177344.716979:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000100:0.0:1786177344.716980:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177344.716981:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1786177344.716984:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 02000000:00000001:1.0:1786177344.716984:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177344.716988:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177344.716990:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.716991:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:0.0:1786177344.716992:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00100000:1.0:1786177344.716993:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177344.716997:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.717000:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177344.717001:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000010:1.0:1786177344.717004:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9500. 00000020:00000040:1.0:1786177344.717007:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000200:0.0:1786177344.717008:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 02000000:00000001:1.0:1786177344.717009:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1786177344.717011:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 02000000:00000001:1.0:1786177344.717012:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.717016:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177344.717017:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1786177344.717018:0:2392:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1786177344.717019:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1786177344.717023:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177344.717025:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af800. 00000100:00000001:1.0:1786177344.717029:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717036:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1786177344.717037:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1786177344.717041:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717047:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717048:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177344.717054:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.717054:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177344.717058:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177344.717061:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177344.717064:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306695424 00000100:00000040:1.0:1786177344.717067:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000cd924b3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1786177344.717069:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5840. 00000100:00000001:0.0:1786177344.717075:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1786177344.717080:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177344.717084:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177344.717088:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177344.717089:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177344.717091:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177344.717094:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1786177344.717096:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306695424 00000100:00080000:1.0:1786177344.717097:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12019 00000100:00000010:0.0:1786177344.717102:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d55a0. 00000400:00000200:1.0:1786177344.717103:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177344.717109:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177344.717114:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1786177344.717115:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:0.0:1786177344.717119:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177344.717119:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1786177344.717124:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1786177344.717124:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.717126:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177344.717128:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.717129:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00100000:0.0:1786177344.717132:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177344/real 1786177344] req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177344.717132:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177344.717133:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177344.717135:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177344.717137:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177344.717139:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177344.717143:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000400:00000001:0.0:1786177344.717149:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177344.717151:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177344.717154:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000100:00000040:1.0:1786177344.717155:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:0.0:1786177344.717156:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177344.717159:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:1.0:1786177344.717159:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1786177344.717163:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:1.0:1786177344.717164:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:0.0:1786177344.717168:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177344.717169:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177344.717176:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000200:0.0:1786177344.717180:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1786177344.717182:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177344.717188:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717190:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1786177344.717191:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1786177344.717193:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177344.717195:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177344.717197:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.717199:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177344.717200:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177344.717202:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717203:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177344.717203:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717205:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1786177344.717208:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8000. 00000100:00100000:0.0:1786177344.717210:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177344.717212:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177344.717216:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177344.717218:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717222:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177344.717222:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177344.717224:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177344.717226:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177344.717228:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1786177344.717230:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad800. 00000100:00000001:1.0:1786177344.717235:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717237:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177344.717240:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.717242:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:1.0:1786177344.717242:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177344.717243:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177344.717250:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177344.717251:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177344.717253:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717255:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:1.0:1786177344.717257:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ab5f277] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1786177344.717260:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1786177344.717268:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177344.717269:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00080000:1.0:1786177344.717271:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1786177344.717273:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:1.0:1786177344.717274:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177344.717277:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177344.717279:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1786177344.717282:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177344.717282:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177344.717287:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12019 00000400:00000200:1.0:1786177344.717293:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177344.717296:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177344.717300:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1786177344.717303:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235605760:192.168.202.132@tcp:250:kworker.0 00000100:00000040:1.0:1786177344.717305:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.717308:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177344.717310:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.717311:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177344.717312:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177344.717313:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177344.717314:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177344.717315:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:0.0:1786177344.717316:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1786177344.717316:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:1.0:1786177344.717321:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000001:0.0:1786177344.717322:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177344.717324:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.717327:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177344.717329:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8380 x1872930235605760/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177344.717332:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177344.717336:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177344.717340:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:0.0:1786177344.717343:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:1.0:1786177344.717346:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 02000000:00000001:0.0:1786177344.717347:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1786177344.717350:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177344.717352:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:1.0:1786177344.717354:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177344.717377:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177344.717380:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177344.717382:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177344.717384:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177344.717386:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:0.0:1786177344.717389:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177344.717391:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177344.717393:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8380. 00000100:00000001:0.0:1786177344.717396:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177344.717397:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.717400:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1786177344.717402:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177344.717406:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717408:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177344.717408:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177344.717412:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1786177344.717413:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1786177344.717417:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:1.0:1786177344.717418:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717420:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177344.717422:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717424:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:1.0:1786177344.717427:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bbb80. 00000100:00000040:0.0:1786177344.717431:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177344.717431:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177344.717435:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177344.717437:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.717441:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177344.717444:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1786177344.717445:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1786177344.717447:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717449:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177344.717453:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:1.0:1786177344.717453:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261aec00. 02000000:00000001:0.0:1786177344.717457:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177344.717459:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177344.717460:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177344.717465:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235605888:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177344.717470:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1786177344.717470:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1786177344.717472:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177344.717474:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177344.717477:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177344.717480:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177344.717482:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177344.717482:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177344.717485:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177344.717488:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:1.0:1786177344.717491:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3691ff3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1786177344.717492:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177344.717496:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177344.717500:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235605888, portal 10 00000100:00000001:1.0:1786177344.717501:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717503:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177344.717505:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487988352 : -110371221563264 : ffff9b9e328b9880) 00000100:00000040:1.0:1786177344.717506:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12029) 00000100:00000040:0.0:1786177344.717515:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717522:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177344.717525:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177344.717528:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177344.717531:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235605888, offset 0 00000400:00000200:0.0:1786177344.717534:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177344.717538:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177344.717542:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177344.717545:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177344.717548:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177344.717550:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177344.717553:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177344.717556:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426800:426800:256:4294967295] 192.168.202.132@tcp LPNI seq info [426762:426762:8:4294967295] 00000400:00000200:0.0:1786177344.717559:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177344.717565:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177344.717569:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177344.717573:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177344.717576:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177344.717579:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177344.717583:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177344.717586:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177344.717589:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177344.717599:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177344.717603:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177344.717606:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177344.717609:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177344.717611:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177344.717613:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177344.717616:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717624:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177344.717627:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717633:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717635:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177344.717638:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177344.717640:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177344.717643:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177344.717644:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306691588 00000100:00000001:0.0:1786177344.717646:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306695424 00000100:00000001:0.0:1786177344.717647:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306691588 00000100:00000001:0.0:1786177344.717649:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306695424 00000100:00000001:0.0:1786177344.717650:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177344.717651:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717652:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177344.717655:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177344/real 1786177344] req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177344.717659:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177344.717660:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177344.717661:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177344.717662:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177344.717663:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177344.717665:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177344.717667:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177344.717675:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717681:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177344.717682:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.717684:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177344.717685:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177344.717689:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717695:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177344.717699:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717706:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177344.717708:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.717710:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177344.717712:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.717714:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177344.717716:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177344.717718:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177344.717722:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177344.717724:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177344.717728:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177344.717736:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235605888:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177344.717740:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177344.717743:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717749:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177344.717751:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177344.717753:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177344.717755:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177344.717756:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177344.717759:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606016:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177344.717762:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177344.717764:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177344.717766:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177344.717768:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177344.717771:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177344.717773:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177344.717776:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177344.717779:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177344.717782:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177344.717784:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235606016, portal 10 00000100:00000001:0.0:1786177344.717786:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177344.717788:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487987456 : -110371221564160 : ffff9b9e328b9500) 00000100:00000040:0.0:1786177344.717793:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717799:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177344.717801:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177344.717803:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177344.717805:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235606016, offset 0 00000400:00000200:0.0:1786177344.717809:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177344.717813:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177344.717816:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177344.717819:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177344.717821:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177344.717823:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177344.717825:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177344.717828:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426801:426801:256:4294967295] 192.168.202.132@tcp LPNI seq info [426763:426763:8:4294967295] 00000400:00000200:0.0:1786177344.717832:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177344.717838:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177344.717842:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177344.717845:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177344.717848:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177344.717851:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177344.717855:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177344.717858:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177344.717861:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177344.717863:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177344.717865:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177344.717869:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177344.717871:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177344.717873:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177344.717875:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177344.717879:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717891:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177344.717893:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717899:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717901:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177344.717903:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177344.717906:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177344.717922:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177344.717925:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306691588 00000100:00000001:0.0:1786177344.717927:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306695424 00000100:00000001:0.0:1786177344.717929:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306691588 00000100:00000001:0.0:1786177344.717931:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306695424 00000100:00000001:0.0:1786177344.717933:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177344.717935:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.717936:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177344.717940:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177344/real 1786177344] req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177344.717947:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177344.717949:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177344.717950:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177344.717953:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177344.717954:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177344.717956:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177344.717959:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177344.717967:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717973:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177344.717975:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.717977:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177344.717978:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177344.717981:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717987:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177344.717991:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.717998:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177344.718000:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718001:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177344.718003:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718005:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177344.718007:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177344.718009:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177344.718012:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177344.718014:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177344.718017:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177344.718024:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606016:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177344.718028:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177344.718031:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718037:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177344.718038:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177344.718040:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177344.718042:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177344.718044:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177344.718047:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606144:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177344.718050:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177344.718051:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177344.718053:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177344.718054:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177344.718057:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177344.718059:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177344.718062:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177344.718065:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177344.718067:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177344.718070:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235606144, portal 4 00000100:00000001:0.0:1786177344.718071:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177344.718073:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487982080 : -110371221569536 : ffff9b9e328b8000) 00000100:00000040:0.0:1786177344.718078:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718083:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177344.718086:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177344.718088:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177344.718090:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235606144, offset 0 00000400:00000200:0.0:1786177344.718093:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177344.718097:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177344.718100:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177344.718103:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177344.718105:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177344.718107:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177344.718110:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177344.718112:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426802:426802:256:4294967295] 192.168.202.132@tcp LPNI seq info [426764:426764:8:4294967295] 00000400:00000200:0.0:1786177344.718116:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177344.718122:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177344.718126:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177344.718129:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177344.718133:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177344.718135:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177344.718139:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177344.718142:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177344.718145:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177344.718147:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177344.718149:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177344.718152:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177344.718155:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177344.718157:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177344.718159:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177344.718163:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718170:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177344.718173:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718179:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.718180:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177344.718183:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177344.718185:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177344.718188:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177344.718190:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306691588 00000100:00000001:0.0:1786177344.718192:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306695424 00000100:00000001:0.0:1786177344.718194:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306691588 00000100:00000001:0.0:1786177344.718196:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306695424 00000100:00000001:0.0:1786177344.718198:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177344.718200:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.718202:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177344.718206:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177344/real 1786177344] req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177344.718212:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177344.718214:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177344.718216:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177344.718218:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177344.718219:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177344.718221:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177344.718224:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177344.718232:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718237:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177344.718238:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718240:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177344.718241:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177344.718245:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718252:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177344.718255:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718261:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177344.718263:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718264:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177344.718265:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718267:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177344.718269:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177344.718271:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177344.718274:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177344.718276:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177344.718280:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177344.718292:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606144:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177344.718296:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718297:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177344.718300:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9880 x1872930235605888/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718306:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177344.718307:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177344.718309:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177344.718312:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177344.718313:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177344.718315:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177344.718317:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177344.718319:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:0.0:1786177344.718321:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177344.718322:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177344.718323:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9880. 00000100:00000001:0.0:1786177344.718326:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177344.718327:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718329:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177344.718332:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9500 x1872930235606016/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718339:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177344.718340:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177344.718341:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177344.718343:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177344.718344:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177344.718345:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177344.718347:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177344.718348:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af800. 02000000:00000001:0.0:1786177344.718350:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177344.718351:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177344.718353:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9500. 00000100:00000001:0.0:1786177344.718356:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177344.718375:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718378:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177344.718381:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8000 x1872930235606144/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718387:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177344.718388:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177344.718390:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177344.718393:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177344.718394:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177344.718395:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177344.718397:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177344.718398:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:0.0:1786177344.718400:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177344.718401:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177344.718402:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8000. 00000100:00000001:0.0:1786177344.718404:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177344.718406:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718407:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718411:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177344.718413:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.718415:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177344.718416:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177344.718418:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177344.718421:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718427:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177344.718428:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177344.718430:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177344.718432:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177344.718434:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177344.718436:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606272:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177344.718439:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177344.718441:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177344.718442:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177344.718444:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177344.718447:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177344.718449:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177344.718452:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177344.718454:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177344.718457:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177344.718460:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235606272, portal 4 00000100:00000001:0.0:1786177344.718461:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177344.718462:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487997312 : -110371221554304 : ffff9b9e328bbb80) 00000100:00000040:0.0:1786177344.718469:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718474:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177344.718476:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177344.718478:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177344.718480:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235606272, offset 0 00000400:00000200:0.0:1786177344.718484:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177344.718487:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177344.718490:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177344.718493:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177344.718496:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177344.718498:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177344.718500:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177344.718503:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426803:426803:256:4294967295] 192.168.202.132@tcp LPNI seq info [426765:426765:8:4294967295] 00000400:00000200:0.0:1786177344.718506:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177344.718512:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177344.718516:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177344.718519:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177344.718523:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177344.718525:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177344.718529:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177344.718531:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177344.718535:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177344.718537:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177344.718539:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177344.718542:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177344.718545:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177344.718546:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177344.718548:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177344.718552:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718559:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177344.718562:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177369 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718568:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.718569:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177344.718573:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177344.718575:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177344.718578:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177344.718580:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306691591 00000100:00000001:0.0:1786177344.718582:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306695424 00000100:00000001:0.0:1786177344.718584:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306691591 00000100:00000001:0.0:1786177344.718586:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306695424 00000100:00000001:0.0:1786177344.718588:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177344.718590:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.718591:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177344.718595:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177344/real 1786177344] req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177344.718602:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177344.718603:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177344.718605:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177344.718607:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177344.718609:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177344.718611:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177344.718614:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177344.718621:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718627:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177344.718628:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718630:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177344.718632:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177344.718635:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718640:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177344.718644:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718650:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177344.718652:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718654:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177344.718655:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718657:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177344.718658:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177344.718660:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177344.718663:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177344.718665:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177344.718669:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177344.718675:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606272:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177344.718679:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718680:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177344.718684:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bbb80 x1872930235606272/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177369 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177344.718690:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177344.718691:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177344.718693:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177344.718695:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177344.718696:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177344.718697:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177344.718699:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177344.718701:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:0.0:1786177344.718703:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177344.718704:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177344.718706:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bbb80. 00000100:00000001:0.0:1786177344.718708:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177344.718710:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718711:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177344.718714:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177344.718715:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177344.718717:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177344.718719:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177345.228419:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177345.228426:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177345.228428:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177345.228434:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177345.228437:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177345.228440:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177345.228442:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177345.228445:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177345.228447:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177345.612446:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177345.612446:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177345.612451:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177345.612453:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177345.612456:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177345.612457:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177345.612460:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177345.612461:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177345.612463:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177345.612466:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177345.612467:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177345.612471:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177345.612473:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177345.612475:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177345.612477:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177345.612517:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177345.612521:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177345.612524:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177346.252413:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177346.252424:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177346.252427:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177346.252431:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177346.252435:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177346.252438:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177346.252441:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177346.252443:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177346.252446:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177346.636454:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177346.636458:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177346.636460:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177346.636462:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177346.636463:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177346.636468:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177346.636469:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177346.636472:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177346.636473:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177346.636474:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177346.636479:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177346.636481:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177346.636483:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177346.636484:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177346.636487:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177346.636542:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177346.636546:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177346.636549:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177347.276421:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177347.276427:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177347.276428:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177347.276433:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177347.276436:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177347.276439:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177347.276441:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177347.276444:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177347.276446:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177347.660427:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177347.660433:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177347.660437:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177347.660439:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177347.660439:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177347.660442:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177347.660444:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177347.660446:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177347.660450:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177347.660454:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177347.660458:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177347.660461:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177347.660464:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177347.660467:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177347.660470:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177347.660496:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177347.660499:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177347.660502:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177348.300419:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177348.300427:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177348.300429:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177348.300434:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177348.300438:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177348.300441:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177348.300444:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177348.300447:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177348.300450:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177348.620464:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306695424, at 4306695488 00000100:00000040:0.0:1786177348.620465:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306695424, at 4306695488 00000100:00000010:0.0:1786177348.620477:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5840. 00000100:00000010:1.0:1786177348.620479:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d55a0. 00000100:00000001:1.0:1786177348.684461:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177348.684464:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177348.684468:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177348.684468:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177348.684471:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177348.684475:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177348.684478:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177348.684480:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177348.684482:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177348.684483:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177348.684486:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177348.684489:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177348.684490:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177348.684496:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177348.684498:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177348.684589:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177348.684593:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177348.684597:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177349.324471:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177349.324480:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177349.324482:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177349.324488:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177349.324492:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177349.324495:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177349.324497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177349.324500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177349.324503:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177349.708500:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177349.708501:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177349.708509:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177349.708517:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177349.708518:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177349.708521:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177349.708523:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177349.708554:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177349.708563:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177349.708569:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177349.708575:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177349.708582:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177349.708586:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177349.708589:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177349.708595:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177349.708607:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177349.708614:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177349.708619:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177349.837035:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177349.837050:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177349.837053:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177349.837060:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177349.837065:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177349.837074:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12024 00000400:00000200:2.0:1786177349.837087:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177349.837094:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177349.837098:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177349.837102:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837107:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177349.837110:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837112:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177349.837118:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177349.837121:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177349.837124:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177349.837129:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177349.837132:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177349.837135:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177349.837138:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837140:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177349.837142:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837144:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177349.837147:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177349.837150:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837153:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177349.837157:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177349.837162:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177349.837166:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837169:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177349.837172:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177349.837189:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000020:00000040:2.0:1786177349.837193:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177349.837198:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177349.837201:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837205:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177349.837210:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177349.837215:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177349.837226:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000001:2.0:1786177349.837235:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177349.837246:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177349.837259:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177349.837266:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177349.837272:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177349.837299:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177349.837303:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177349.837307:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177349.837309:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177349.837310:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177349.837314:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177349.837317:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12024 00000400:00000200:2.0:1786177349.837321:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177349.837324:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177349.837332:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177349.837335:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837338:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177349.837342:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837344:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177349.837347:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177349.837348:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177349.837351:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177349.837353:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000100:00000001:0.0:1786177349.837355:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1786177349.837356:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000001:0.0:1786177349.837389:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1786177349.837391:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177349.837394:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177349.837403:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1786177349.837411:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177349.837413:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177349.837416:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837422:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177349.837427:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00000001:0.0:1786177349.837430:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1786177349.837432:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177349.837435:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837437:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177349.837441:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177349.837442:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177349.837447:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177349.837448:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177349.837453:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177349.837453:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837459:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:0.0:1786177349.837462:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235606400:192.168.202.132@tcp:250:kworker.0 00000008:00100000:2.0:1786177349.837466:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1786177349.837469:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1786177349.837471:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837475:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177349.837477:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:2.0:1786177349.837479:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 02000000:00000001:0.0:1786177349.837481:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1786177349.837484:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786177349.837486:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1786177349.837488:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177349.837491:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837493:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:0.0:1786177349.837494:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:2.0:1786177349.837495:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177349.837499:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177349.837499:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177349.837508:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000400:00000200:0.0:1786177349.837512:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177349.837514:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1786177349.837520:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000100:00100000:2.0:1786177349.837522:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177349.837532:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177349.837535:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177349.837539:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1786177349.837540:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177349.837550:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235606400, portal 25 00000100:00000001:0.0:1786177349.837556:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177349.837558:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469625472 : -110371239926144 : ffff9b9e31736680) 00000100:00000001:2.0:1786177349.837561:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177349.837565:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1786177349.837569:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177349.837570:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177349.837574:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177349.837577:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177349.837584:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1786177349.837585:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1786177349.837588:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12024 00000100:00000040:0.0:1786177349.837592:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177349.837597:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000400:00000200:2.0:1786177349.837597:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000200:0.0:1786177349.837603:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235606400, offset 0 00000400:00000200:2.0:1786177349.837604:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177349.837610:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1786177349.837615:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00000040:2.0:1786177349.837618:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837621:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177349.837624:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837626:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:0.0:1786177349.837628:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000020:00000001:2.0:1786177349.837630:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177349.837633:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177349.837635:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:0.0:1786177349.837639:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000020:00000040:2.0:1786177349.837640:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177349.837645:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177349.837648:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000400:00000200:0.0:1786177349.837649:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1786177349.837653:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:0.0:1786177349.837657:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00000001:2.0:1786177349.837658:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177349.837662:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000200:0.0:1786177349.837664:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1786177349.837666:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:0.0:1786177349.837670:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000020:00000040:2.0:1786177349.837671:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177349.837679:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000400:00000200:0.0:1786177349.837680:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426804:426804:256:4294967295] 192.168.202.132@tcp LPNI seq info [426766:426766:8:4294967295] 00000100:00000001:2.0:1786177349.837685:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177349.837688:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1786177349.837689:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:2.0:1786177349.837692:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177349.837696:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837698:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177349.837702:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1786177349.837705:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000008:00000001:2.0:1786177349.837706:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837711:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177349.837714:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 00000020:00000040:2.0:1786177349.837719:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000800:00000200:0.0:1786177349.837723:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 02000000:00000001:2.0:1786177349.837724:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177349.837727:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837729:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177349.837730:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177349.837732:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177349.837735:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:2.0:1786177349.837740:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1786177349.837741:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000100:00100000:2.0:1786177349.837748:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1786177349.837754:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177349.837763:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1786177349.837763:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177349.837767:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1786177349.837773:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:2.0:1786177349.837774:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1786177349.837779:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1786177349.837789:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1786177349.837790:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00080000:2.0:1786177349.837792:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:0.0:1786177349.837795:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000100:00080000:2.0:1786177349.837798:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177349.837801:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177349.837803:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1786177349.837804:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1786177349.837809:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1786177349.837813:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00080000:2.0:1786177349.837816:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12024 00000400:00000200:2.0:1786177349.837822:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177349.837823:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000200:2.0:1786177349.837827:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000400:00000010:0.0:1786177349.837828:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00080000:2.0:1786177349.837831:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177349.837835:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177349.837836:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1786177349.837837:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177349.837843:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000200:0.0:1786177349.837844:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177349.837846:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177349.837852:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177349.837854:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177349.837869:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177349.837871:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177349.837874:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177349.837876:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177349.837880:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837885:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177349.837889:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837893:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177349.837898:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00000001:0.0:1786177349.837901:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1786177349.837902:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177349.837906:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177349.837908:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177349.837909:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177349.837913:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177349.837918:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837922:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177349.837925:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1786177349.837926:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177349.837931:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837933:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177349.837936:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000020:00000040:2.0:1786177349.837939:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000100:00000001:0.0:1786177349.837941:0:2392:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1786177349.837942:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177349.837946:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.837948:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177349.837950:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1786177349.837951:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177349.837952:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177349.837959:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000400:00000200:0.0:1786177349.837962:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177349.837964:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.837967:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306700545 00000100:00100000:2.0:1786177349.837972:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:0.0:1786177349.837976:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5840. 00000100:00000001:2.0:1786177349.837984:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177349.837986:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177349.837988:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177349.837994:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838001:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306700545 00000100:00000001:2.0:1786177349.838002:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177349.838008:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5a80. 00000100:00080000:2.0:1786177349.838008:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177349.838014:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786177349.838015:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177349.838016:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177349.838020:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177349.838021:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177349.838024:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1786177349.838026:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177349.838029:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12024 00000100:00000001:0.0:1786177349.838032:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1786177349.838036:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00100000:0.0:1786177349.838041:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177349/real 1786177349] req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177349.838042:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177349.838047:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177349.838054:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.838059:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1786177349.838063:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1786177349.838063:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000001:0.0:1786177349.838068:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1786177349.838068:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000010:0.0:1786177349.838072:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000020:00000001:2.0:1786177349.838073:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177349.838076:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000400:00000200:0.0:1786177349.838079:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000020:00000001:2.0:1786177349.838080:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:0.0:1786177349.838084:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000020:00000040:2.0:1786177349.838085:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000001:0.0:1786177349.838090:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1786177349.838090:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177349.838094:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177349.838098:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.838100:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177349.838102:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177349.838104:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177349.838106:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000200:0.0:1786177349.838111:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1786177349.838112:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177349.838118:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177349.838123:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177349.838127:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1786177349.838132:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177349.838132:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.838136:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177349.838139:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177349.838145:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.838147:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177349.838148:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.838152:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1786177349.838154:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000100:00000001:0.0:1786177349.838157:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:2.0:1786177349.838159:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000100:00000001:0.0:1786177349.838160:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1786177349.838164:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1786177349.838168:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1786177349.838168:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177349.838171:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177349.838174:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177349.838177:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.838183:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1786177349.838184:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:2.0:1786177349.838189:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177349.838190:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177349.838198:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177349.838201:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177349.838206:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.838210:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177349.838213:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177349.838215:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177349.838217:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177349.838218:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.838221:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177349.838226:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1786177349.838226:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1786177349.838235:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177349.838241:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1786177349.838242:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177349.838248:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177349.838249:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12034) 00000100:00100000:0.0:1786177349.838261:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235606400:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177349.838267:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.838269:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177349.838273:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736680 x1872930235606400/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838280:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177349.838282:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177349.838285:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177349.838289:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177349.838291:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177349.838293:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177349.838295:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177349.838298:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:0.0:1786177349.838302:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177349.838304:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177349.838306:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736680. 00000100:00000001:0.0:1786177349.838309:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177349.838310:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.838313:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.838320:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177349.838323:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.838325:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177349.838327:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177349.838328:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177349.838331:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838338:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177349.838339:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177349.838341:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177349.838343:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177349.838346:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177349.838349:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606528:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177349.838352:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177349.838354:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177349.838356:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177349.838374:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177349.838378:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177349.838380:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177349.838384:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177349.838387:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177349.838391:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177349.838394:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235606528, portal 10 00000100:00000001:0.0:1786177349.838395:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177349.838397:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469626368 : -110371239925248 : ffff9b9e31736a00) 00000100:00000040:0.0:1786177349.838404:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838411:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177349.838413:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177349.838415:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177349.838418:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235606528, offset 0 00000400:00000200:0.0:1786177349.838421:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177349.838425:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177349.838429:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177349.838431:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177349.838434:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177349.838437:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177349.838439:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177349.838442:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426805:426805:256:4294967295] 192.168.202.132@tcp LPNI seq info [426767:426767:8:4294967295] 00000400:00000200:0.0:1786177349.838445:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177349.838452:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177349.838456:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177349.838459:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177349.838463:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177349.838466:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177349.838470:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177349.838472:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177349.838475:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177349.838477:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177349.838479:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177349.838483:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177349.838486:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177349.838488:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177349.838490:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177349.838494:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838500:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177349.838503:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838515:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.838517:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177349.838519:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177349.838523:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177349.838526:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177349.838528:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306696709 00000100:00000001:0.0:1786177349.838531:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306700545 00000100:00000001:0.0:1786177349.838533:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306696709 00000100:00000001:0.0:1786177349.838534:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306700545 00000100:00000001:0.0:1786177349.838537:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177349.838539:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.838541:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177349.838544:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177349/real 1786177349] req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177349.838552:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177349.838553:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177349.838555:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177349.838557:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177349.838558:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177349.838560:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177349.838563:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177349.838571:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838577:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177349.838578:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.838580:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177349.838581:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177349.838585:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838592:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177349.838596:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838602:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177349.838604:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.838605:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177349.838606:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.838608:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177349.838610:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177349.838612:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177349.838615:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177349.838617:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177349.838621:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177349.838628:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606528:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177349.838632:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177349.838635:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838640:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177349.838642:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177349.838643:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177349.838645:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177349.838647:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177349.838650:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606656:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177349.838652:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177349.838654:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177349.838656:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177349.838658:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177349.838661:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177349.838663:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177349.838666:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177349.838668:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177349.838671:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177349.838673:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235606656, portal 10 00000100:00000001:0.0:1786177349.838675:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177349.838676:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469615616 : -110371239936000 : ffff9b9e31734000) 00000100:00000040:0.0:1786177349.838681:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838687:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177349.838689:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177349.838691:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177349.838693:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235606656, offset 0 00000400:00000200:0.0:1786177349.838697:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177349.838700:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177349.838703:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177349.838706:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177349.838709:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177349.838711:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177349.838713:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177349.838716:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426806:426806:256:4294967295] 192.168.202.132@tcp LPNI seq info [426768:426768:8:4294967295] 00000400:00000200:0.0:1786177349.838720:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177349.838727:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177349.838731:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177349.838734:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177349.838738:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177349.838741:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177349.838745:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177349.838748:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177349.838751:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177349.838754:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177349.838757:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177349.838760:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177349.838762:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177349.838764:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177349.838766:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177349.838769:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838777:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177349.838780:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838786:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.838788:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177349.838791:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177349.838793:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177349.838796:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177349.838797:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306696709 00000100:00000001:0.0:1786177349.838800:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306700545 00000100:00000001:0.0:1786177349.838801:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306696709 00000100:00000001:0.0:1786177349.838803:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306700545 00000100:00000001:0.0:1786177349.838806:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177349.838808:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.838810:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177349.838814:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177349/real 1786177349] req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177349.838822:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177349.838823:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177349.838825:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177349.838827:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177349.838828:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177349.838831:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177349.838834:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177349.838841:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838848:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177349.838849:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.838851:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177349.838852:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177349.838855:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838876:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177349.838881:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838887:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177349.838890:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.838891:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177349.838892:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.838894:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177349.838895:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177349.838897:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177349.838900:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177349.838902:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177349.838906:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177349.838913:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606656:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177349.838916:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177349.838919:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838925:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177349.838927:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177349.838929:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177349.838931:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177349.838932:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177349.838935:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606784:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177349.838938:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177349.838939:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177349.838941:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177349.838943:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177349.838946:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177349.838948:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177349.838951:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177349.838953:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177349.838957:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177349.838959:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235606784, portal 4 00000100:00000001:0.0:1786177349.838961:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177349.838962:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469624576 : -110371239927040 : ffff9b9e31736300) 00000100:00000040:0.0:1786177349.838967:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.838973:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177349.838975:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177349.838978:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177349.838980:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235606784, offset 0 00000400:00000200:0.0:1786177349.838983:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177349.838987:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177349.838990:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177349.838992:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177349.838994:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177349.838997:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177349.838999:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177349.839003:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426807:426807:256:4294967295] 192.168.202.132@tcp LPNI seq info [426769:426769:8:4294967295] 00000400:00000200:0.0:1786177349.839006:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177349.839012:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177349.839016:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177349.839020:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177349.839023:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177349.839025:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177349.839029:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177349.839031:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177349.839034:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177349.839036:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177349.839039:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177349.839042:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177349.839045:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177349.839047:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177349.839049:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177349.839053:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839060:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177349.839063:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839069:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.839071:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177349.839073:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177349.839076:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177349.839079:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177349.839081:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306696709 00000100:00000001:0.0:1786177349.839084:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306700545 00000100:00000001:0.0:1786177349.839086:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306696709 00000100:00000001:0.0:1786177349.839088:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306700545 00000100:00000001:0.0:1786177349.839090:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177349.839092:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.839094:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177349.839097:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177349/real 1786177349] req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177349.839103:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177349.839104:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177349.839106:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177349.839108:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177349.839110:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177349.839111:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177349.839114:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177349.839123:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839129:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177349.839131:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839133:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177349.839134:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177349.839137:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839143:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177349.839146:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839153:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177349.839155:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839157:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177349.839158:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839160:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177349.839161:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177349.839163:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177349.839167:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177349.839169:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177349.839172:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177349.839179:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606784:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177349.839183:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177349.839186:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839191:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177349.839193:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177349.839195:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177349.839197:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177349.839198:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177349.839201:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606912:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177349.839204:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177349.839206:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177349.839207:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177349.839209:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177349.839212:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177349.839214:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177349.839217:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177349.839219:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177349.839222:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177349.839225:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235606912, portal 4 00000100:00000001:0.0:1786177349.839226:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177349.839227:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469616512 : -110371239935104 : ffff9b9e31734380) 00000100:00000040:0.0:1786177349.839233:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839239:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177349.839242:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177349.839244:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177349.839257:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235606912, offset 0 00000400:00000200:0.0:1786177349.839261:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177349.839265:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177349.839268:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177349.839271:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177349.839273:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177349.839275:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177349.839278:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177349.839281:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426808:426808:256:4294967295] 192.168.202.132@tcp LPNI seq info [426770:426770:8:4294967295] 00000400:00000200:0.0:1786177349.839285:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177349.839290:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177349.839295:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177349.839298:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177349.839302:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177349.839305:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177349.839309:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177349.839311:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177349.839316:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177349.839318:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177349.839321:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177349.839323:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177349.839326:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177349.839328:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177349.839330:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177349.839334:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839341:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177349.839344:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177374 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839351:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.839352:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177349.839355:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177349.839379:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177349.839383:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177349.839385:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306696712 00000100:00000001:0.0:1786177349.839386:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306700545 00000100:00000001:0.0:1786177349.839388:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306696712 00000100:00000001:0.0:1786177349.839390:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306700545 00000100:00000001:0.0:1786177349.839392:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177349.839394:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.839395:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177349.839399:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177349/real 1786177349] req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177349.839406:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177349.839408:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177349.839409:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177349.839412:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177349.839413:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177349.839415:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177349.839418:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177349.839426:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839432:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177349.839433:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839435:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177349.839436:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177349.839440:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839446:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177349.839449:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839456:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177349.839459:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839460:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177349.839461:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839463:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177349.839464:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177349.839467:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177349.839470:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177349.839472:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177349.839476:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177349.839484:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235606912:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177349.839488:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839490:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177349.839493:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736a00 x1872930235606528/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839499:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177349.839501:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177349.839503:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177349.839505:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177349.839507:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177349.839508:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177349.839510:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177349.839512:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:0.0:1786177349.839515:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177349.839516:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177349.839518:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736a00. 00000100:00000001:0.0:1786177349.839521:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177349.839522:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839524:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177349.839527:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734000 x1872930235606656/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839533:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177349.839534:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177349.839535:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177349.839537:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177349.839538:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177349.839539:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177349.839541:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177349.839542:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:0.0:1786177349.839544:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177349.839545:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177349.839547:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734000. 00000100:00000001:0.0:1786177349.839549:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177349.839550:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839552:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177349.839555:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736300 x1872930235606784/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839561:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177349.839562:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177349.839563:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177349.839565:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177349.839566:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177349.839567:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177349.839569:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177349.839570:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:0.0:1786177349.839572:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177349.839573:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177349.839575:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736300. 00000100:00000001:0.0:1786177349.839577:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177349.839578:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839580:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177349.839583:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734380 x1872930235606912/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177374 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177349.839594:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177349.839595:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177349.839597:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177349.839599:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177349.839600:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177349.839601:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177349.839603:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177349.839605:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:0.0:1786177349.839607:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177349.839608:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177349.839609:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734380. 00000100:00000001:0.0:1786177349.839611:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177349.839612:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839614:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177349.839619:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177349.839620:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177349.839623:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177349.839624:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177350.348456:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177350.348466:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177350.348468:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177350.348474:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177350.348479:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177350.348483:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177350.348485:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177350.348487:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177350.348491:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177350.732530:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177350.732537:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177350.732543:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177350.732546:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177350.732548:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177350.732552:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177350.732555:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177350.732557:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177350.732559:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177350.732561:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177350.732565:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177350.732566:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177350.732756:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177350.732761:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177350.732763:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177350.732768:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177350.732786:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177350.732790:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177350.732793:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177350.732796:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177350.732799:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177350.732903:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177350.732907:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177350.732911:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786177351.308503:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786177351.308512:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786177351.308518:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000001:0.0:1786177351.308695:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000200:1.0:1786177351.308704:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1786177351.308711:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786177351.308718:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786177351.308728:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177351.372571:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177351.372579:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177351.372582:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177351.372587:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177351.372591:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177351.372594:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177351.372597:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177351.372599:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177351.372602:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177351.756496:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177351.756505:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177351.756512:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177351.756513:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177351.756517:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177351.756518:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177351.756780:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177351.756784:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177351.756788:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177351.756916:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177351.756921:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177351.756923:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177351.756928:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177351.756931:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177351.756935:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177351.756938:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177351.756940:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177351.756943:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177352.397084:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177352.397094:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177352.397097:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177352.397104:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177352.397109:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177352.397114:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177352.397117:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177352.397120:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177352.397123:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177352.780497:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177352.780504:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177352.780510:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177352.780512:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177352.780515:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177352.780516:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177352.782644:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177352.782649:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177352.782653:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177352.785868:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177352.785875:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177352.785878:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177352.785883:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177352.785887:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177352.785891:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177352.785894:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177352.785896:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177352.785900:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177353.424752:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177353.424759:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177353.424760:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177353.424764:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177353.424767:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177353.424770:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177353.424773:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177353.424776:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177353.424779:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177353.740526:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306700545, at 4306700608 00000100:00000010:0.0:1786177353.740541:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5840. 00000100:00000040:1.0:1786177353.740850:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306700545, at 4306700608 00000100:00000010:1.0:1786177353.740857:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5a80. 00000100:00000001:3.0:1786177353.804434:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177353.804442:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177353.804445:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177353.804450:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177353.804454:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177353.804458:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177353.804461:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177353.804462:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177353.804465:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177353.804469:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1786177353.804469:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177353.804473:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177353.804477:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177353.804481:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177353.804483:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177353.804486:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177353.804488:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177353.804492:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177353.804493:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177353.804497:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177353.804498:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177353.804500:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177353.804501:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177353.804503:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177353.804504:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177353.804506:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177353.804508:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177353.804556:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177353.804618:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177353.804621:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177354.444546:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177354.444559:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177354.444562:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177354.444570:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177354.444574:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177354.444580:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177354.444584:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177354.444587:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177354.444592:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177354.828470:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177354.828476:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1786177354.828478:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177354.828481:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177354.828482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177354.828489:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177354.828490:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177354.828493:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177354.828493:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177354.828499:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177354.828501:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177354.828503:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177354.828503:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177354.828507:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177354.828511:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177354.828551:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177354.828554:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177354.828557:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177354.956454:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177354.956463:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177354.956466:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177354.956469:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177354.956474:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177354.956480:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12029 00000400:00000200:1.0:1786177354.956488:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177354.956494:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177354.956506:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177354.956509:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.956512:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177354.956515:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.956517:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177354.956521:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177354.956523:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177354.956525:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177354.956528:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177354.956531:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177354.956533:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177354.956536:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.956537:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177354.956539:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.956540:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177354.956543:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177354.956546:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177354.956548:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177354.956550:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177354.956554:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177354.956556:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.956558:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177354.956561:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177354.956568:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bad80. 00000020:00000040:1.0:1786177354.956571:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177354.956575:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177354.956577:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.956580:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177354.956583:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177354.956587:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177354.956595:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261aec00. 00000100:00000001:1.0:1786177354.956602:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177354.956612:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177354.956622:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177354.956627:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177354.956632:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b748f214] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177354.956765:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177354.956768:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177354.956771:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177354.956773:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177354.956774:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177354.956777:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1786177354.956875:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177354.956878:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177354.956880:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177354.956885:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.956891:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177354.956893:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177354.956895:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177354.956897:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177354.956899:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177354.956901:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235607040:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177354.956904:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177354.956907:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177354.956909:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177354.956911:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177354.956915:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177354.956916:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177354.956920:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177354.956923:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177354.956928:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177354.956933:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235607040, portal 25 00000100:00000001:0.0:1786177354.956935:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177354.956936:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487993728 : -110371221557888 : ffff9b9e328bad80) 00000100:00000040:0.0:1786177354.956939:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.956944:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177354.956946:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177354.956948:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177354.956959:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235607040, offset 0 00000400:00000200:0.0:1786177354.956963:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177354.956969:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177354.956974:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177354.956978:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177354.956981:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177354.956983:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177354.956985:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177354.956988:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426809:426809:256:4294967295] 192.168.202.132@tcp LPNI seq info [426771:426771:8:4294967295] 00000400:00000200:0.0:1786177354.956991:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177354.957002:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177354.957006:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177354.957010:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177354.957015:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177354.957018:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1786177354.957021:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177354.957024:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1786177354.957026:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177354.957028:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177354.957032:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177354.957035:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177354.957039:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177354.957041:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177354.957043:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177354.957046:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.957052:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177354.957054:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.957059:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.957060:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177354.957065:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177354.957069:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177354.957072:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306705664 00000100:00000010:0.0:1786177354.957076:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5060. 00000100:00000001:0.0:1786177354.957083:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177354.957089:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306705664 00000100:00000010:0.0:1786177354.957092:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5a80. 00000100:00000001:0.0:1786177354.957095:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177354.957098:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177354.957100:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.957103:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177354.957108:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177354/real 1786177354] req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177354.957116:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177354.957117:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177354.957119:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177354.957121:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177354.957122:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177354.957123:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177354.957126:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177354.957131:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.957135:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177354.957136:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.957137:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177354.957139:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177354.957141:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.957145:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177354.957148:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.957152:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177354.957154:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.957156:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177354.957156:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.957158:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177354.957160:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177354.957162:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177354.957164:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177354.957165:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177354.957168:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177354.957172:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235607040:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177354.957175:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.957177:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177354.957179:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bad80 x1872930235607040/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.957183:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177354.957185:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177354.957186:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177354.957188:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177354.957189:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177354.957190:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177354.957192:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177354.957193:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:0.0:1786177354.957196:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177354.957197:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177354.957198:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bad80. 00000100:00000001:0.0:1786177354.957201:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177354.957202:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.957204:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.957208:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177354.957210:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.957211:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177354.957212:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177354.957243:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12029 00000400:00000200:1.0:1786177354.957246:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177354.957247:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177354.957249:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177354.957251:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957253:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177354.957254:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957255:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177354.957257:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177354.957258:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177354.957259:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177354.957260:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177354.957262:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177354.957263:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177354.957264:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957265:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177354.957267:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957268:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177354.957270:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:1.0:1786177354.957271:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957273:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177354.957274:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177354.957277:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177354.957278:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957279:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177354.957283:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177354.957284:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957285:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177354.957288:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328baa00. 00000020:00000040:1.0:1786177354.957289:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177354.957291:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177354.957292:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957294:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177354.957295:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177354.957296:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177354.957299:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad800. 00000100:00000001:1.0:1786177354.957301:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177354.957304:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177354.957309:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177354.957311:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177354.957313:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ebce398] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177354.957325:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177354.957327:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177354.957329:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177354.957330:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177354.957331:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177354.957333:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177354.957335:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12029 00000400:00000200:1.0:1786177354.957337:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177354.957338:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177354.957340:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177354.957341:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957343:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177354.957344:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957345:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177354.957346:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177354.957347:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177354.957347:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177354.957348:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177354.957349:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177354.957351:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177354.957352:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957354:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177354.957355:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957386:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177354.957388:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:1.0:1786177354.957389:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957391:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177354.957392:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177354.957394:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177354.957395:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957396:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177354.957398:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177354.957399:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957400:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177354.957403:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8a80. 00000020:00000040:1.0:1786177354.957404:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177354.957405:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177354.957406:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957407:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177354.957408:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177354.957409:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177354.957412:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af800. 00000100:00000001:1.0:1786177354.957414:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177354.957417:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177354.957421:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177354.957422:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177354.957424:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002bb61190] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177354.957429:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177354.957430:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177354.957432:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177354.957433:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177354.957434:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177354.957435:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177354.957437:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12029 00000400:00000200:1.0:1786177354.957439:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177354.957440:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177354.957441:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177354.957443:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957444:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177354.957445:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957447:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177354.957448:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177354.957449:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177354.957450:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177354.957451:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177354.957452:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177354.957453:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:1.0:1786177354.957455:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957456:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177354.957457:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957458:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177354.957459:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:1.0:1786177354.957460:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957461:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177354.957462:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177354.957464:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177354.957465:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957466:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177354.957467:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177354.957468:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957469:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177354.957470:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9180. 00000020:00000040:1.0:1786177354.957471:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177354.957473:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177354.957473:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957474:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177354.957475:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177354.957475:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177354.957477:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae000. 00000100:00000001:1.0:1786177354.957479:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177354.957481:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177354.957485:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177354.957486:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177354.957488:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d01dc3b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177354.957492:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177354.957494:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177354.957502:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177354.957503:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177354.957504:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177354.957505:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177354.957507:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12029 00000400:00000200:1.0:1786177354.957509:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177354.957511:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177354.957512:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177354.957514:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957515:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177354.957516:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957517:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177354.957518:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177354.957519:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177354.957520:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177354.957521:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:1.0:1786177354.957522:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:1.0:1786177354.957523:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:1.0:1786177354.957524:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957525:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177354.957526:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957527:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177354.957528:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:1.0:1786177354.957530:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177354.957531:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177354.957532:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177354.957534:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177354.957535:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957536:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177354.957537:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177354.957538:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957539:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177354.957541:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb800. 00000020:00000040:1.0:1786177354.957542:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177354.957543:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177354.957544:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177354.957545:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177354.957545:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177354.957546:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177354.957548:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad400. 00000100:00000001:1.0:1786177354.957550:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177354.957552:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177354.957556:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177354.957557:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177354.957559:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008148c2a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177354.957563:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177354.957565:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12039) 00000100:00000001:0.0:1786177354.958046:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177354.958050:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177354.958051:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177354.958056:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958064:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177354.958065:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177354.958067:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177354.958070:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177354.958072:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177354.958075:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607168:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177354.958078:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177354.958080:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177354.958081:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177354.958083:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177354.958087:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177354.958089:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177354.958093:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177354.958095:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177354.958099:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177354.958102:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235607168, portal 10 00000100:00000001:0.0:1786177354.958104:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177354.958105:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487992832 : -110371221558784 : ffff9b9e328baa00) 00000100:00000040:0.0:1786177354.958111:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958117:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177354.958119:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177354.958121:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177354.958124:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235607168, offset 0 00000400:00000200:0.0:1786177354.958128:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177354.958132:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177354.958136:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177354.958139:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177354.958142:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177354.958144:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177354.958146:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177354.958149:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426810:426810:256:4294967295] 192.168.202.132@tcp LPNI seq info [426772:426772:8:4294967295] 00000400:00000200:0.0:1786177354.958153:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177354.958159:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177354.958162:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177354.958165:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177354.958169:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177354.958172:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1786177354.958176:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177354.958178:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1786177354.958181:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177354.958183:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177354.958186:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177354.958189:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177354.958192:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177354.958194:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177354.958196:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177354.958200:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958207:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177354.958211:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958216:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.958218:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177354.958222:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177354.958225:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177354.958229:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177354.958231:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306701828 00000100:00000001:0.0:1786177354.958233:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306705664 00000100:00000001:0.0:1786177354.958235:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306701828 00000100:00000001:0.0:1786177354.958238:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306705664 00000100:00000001:0.0:1786177354.958240:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177354.958242:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.958244:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177354.958247:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177354/real 1786177354] req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177354.958255:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177354.958257:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177354.958258:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177354.958260:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177354.958262:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177354.958264:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177354.958267:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177354.958275:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958282:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177354.958283:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.958286:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177354.958287:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177354.958291:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958297:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177354.958301:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958308:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177354.958310:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.958312:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177354.958313:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.958315:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177354.958317:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177354.958320:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177354.958323:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177354.958325:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177354.958329:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177354.958337:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607168:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177354.958341:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177354.958344:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958350:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177354.958352:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177354.958354:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177354.958387:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177354.958390:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177354.958393:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607296:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177354.958396:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177354.958399:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177354.958400:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177354.958402:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177354.958405:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177354.958407:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177354.958410:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177354.958413:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177354.958416:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177354.958418:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235607296, portal 10 00000100:00000001:0.0:1786177354.958420:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177354.958421:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487984768 : -110371221566848 : ffff9b9e328b8a80) 00000100:00000040:0.0:1786177354.958427:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958434:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177354.958435:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177354.958438:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177354.958440:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235607296, offset 0 00000400:00000200:0.0:1786177354.958444:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177354.958447:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177354.958451:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177354.958454:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177354.958457:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177354.958459:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177354.958461:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177354.958464:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426811:426811:256:4294967295] 192.168.202.132@tcp LPNI seq info [426773:426773:8:4294967295] 00000400:00000200:0.0:1786177354.958468:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177354.958474:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177354.958479:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177354.958482:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177354.958486:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177354.958489:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1786177354.958492:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177354.958495:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1786177354.958512:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177354.958513:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177354.958516:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177354.958520:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177354.958522:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177354.958524:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177354.958527:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177354.958530:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958538:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177354.958541:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958552:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.958554:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177354.958557:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177354.958560:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177354.958563:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177354.958565:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306701828 00000100:00000001:0.0:1786177354.958567:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306705664 00000100:00000001:0.0:1786177354.958569:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306701828 00000100:00000001:0.0:1786177354.958571:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306705664 00000100:00000001:0.0:1786177354.958573:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177354.958574:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.958576:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177354.958580:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177354/real 1786177354] req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177354.958588:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177354.958589:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177354.958591:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177354.958593:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177354.958595:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177354.958597:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177354.958600:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177354.958608:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958615:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177354.958616:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.958618:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177354.958619:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177354.958622:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.958630:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177354.958633:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959127:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177354.959129:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.959131:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177354.959132:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.959133:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177354.959135:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177354.959137:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177354.959140:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177354.959141:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177354.959144:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177354.959149:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607296:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177354.959152:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177354.959154:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959158:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177354.959159:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177354.959160:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177354.959162:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177354.959163:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177354.959165:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607424:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177354.959167:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177354.959168:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177354.959170:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177354.959171:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177354.959174:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177354.959175:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177354.959177:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177354.959179:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177354.959182:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177354.959184:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235607424, portal 4 00000100:00000001:0.0:1786177354.959185:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177354.959186:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487986560 : -110371221565056 : ffff9b9e328b9180) 00000100:00000040:0.0:1786177354.959189:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959285:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177354.959287:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177354.959289:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177354.959292:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235607424, offset 0 00000400:00000200:0.0:1786177354.959294:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177354.959297:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177354.959300:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177354.959302:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177354.959304:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177354.959305:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177354.959307:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177354.959309:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426812:426812:256:4294967295] 192.168.202.132@tcp LPNI seq info [426774:426774:8:4294967295] 00000400:00000200:0.0:1786177354.959311:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177354.959316:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177354.959318:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177354.959320:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177354.959323:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177354.959325:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1786177354.959327:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177354.959329:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1786177354.959331:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177354.959333:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177354.959335:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177354.959337:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177354.959339:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177354.959340:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177354.959342:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177354.959344:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959349:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177354.959351:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959355:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.959454:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177354.959457:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177354.959459:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177354.959461:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177354.959463:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306701830 00000100:00000001:0.0:1786177354.959465:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306705664 00000100:00000001:0.0:1786177354.959466:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306701830 00000100:00000001:0.0:1786177354.959467:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306705664 00000100:00000001:0.0:1786177354.959469:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177354.959470:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.959471:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177354.959474:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177354/real 1786177354] req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177354.959478:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177354.959479:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177354.959480:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177354.959482:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177354.959483:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177354.959485:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177354.959487:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177354.959491:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959495:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177354.959496:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.959497:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177354.959498:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177354.959500:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959503:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177354.959505:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959509:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177354.959511:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.959512:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177354.959512:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.959513:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177354.959514:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177354.959516:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177354.959517:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177354.959518:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177354.959520:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177354.959525:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607424:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177354.959527:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177354.959528:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959533:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177354.959534:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177354.959535:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177354.959536:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177354.959537:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177354.959539:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607552:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177354.959540:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177354.959541:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177354.959542:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177354.959543:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177354.959545:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177354.959546:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177354.959548:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177354.959550:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177354.959551:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177354.959553:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235607552, portal 4 00000100:00000001:0.0:1786177354.959554:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177354.959555:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487996416 : -110371221555200 : ffff9b9e328bb800) 00000100:00000040:0.0:1786177354.959559:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959562:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177354.959563:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177354.959565:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177354.959566:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235607552, offset 0 00000400:00000200:0.0:1786177354.959568:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177354.959570:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177354.959572:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177354.959574:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177354.959575:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177354.959577:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177354.959578:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177354.959580:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426813:426813:256:4294967295] 192.168.202.132@tcp LPNI seq info [426775:426775:8:4294967295] 00000400:00000200:0.0:1786177354.959582:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177354.959586:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177354.959589:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177354.959591:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177354.959593:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177354.959595:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1786177354.959598:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177354.959599:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 12 secs later) 00000800:00000100:0.0:1786177354.959601:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177354.959603:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177354.959604:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177354.959606:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177354.959608:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177354.959609:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177354.959610:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177354.959612:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959616:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177354.959618:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177379 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959622:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.959622:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177354.959624:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177354.959626:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177354.959629:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177354.959630:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306701832 00000100:00000001:0.0:1786177354.959632:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306705664 00000100:00000001:0.0:1786177354.959634:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306701832 00000100:00000001:0.0:1786177354.959636:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306705664 00000100:00000001:0.0:1786177354.959887:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177354.959888:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.959890:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177354.959894:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177354/real 1786177354] req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177354.959899:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177354.959900:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177354.959901:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177354.959903:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177354.959904:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177354.959906:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177354.959908:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177354.959912:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959916:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177354.959917:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.959917:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177354.959918:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177354.959920:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959924:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177354.959926:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959930:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177354.959932:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.959932:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177354.959933:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.959934:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177354.959935:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177354.959936:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177354.959939:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177354.959940:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177354.959942:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177354.959946:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607552:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177354.959957:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.959959:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177354.959961:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328baa00 x1872930235607168/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959965:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177354.959966:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177354.959968:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177354.959970:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177354.959971:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177354.959972:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177354.959974:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177354.959975:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:0.0:1786177354.959977:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177354.959979:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177354.959980:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328baa00. 00000100:00000001:0.0:1786177354.959982:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177354.959983:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.959984:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177354.959986:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8a80 x1872930235607296/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.959990:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177354.959991:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177354.959992:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177354.959993:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177354.959993:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177354.959994:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177354.959995:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177354.960001:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af800. 02000000:00000001:0.0:1786177354.960002:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177354.960003:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177354.960004:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8a80. 00000100:00000001:0.0:1786177354.960005:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177354.960006:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.960007:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177354.960009:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9180 x1872930235607424/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.960012:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177354.960013:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177354.960014:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177354.960015:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177354.960015:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177354.960016:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177354.960017:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177354.960018:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae000. 02000000:00000001:0.0:1786177354.960019:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177354.960019:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177354.960020:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9180. 00000100:00000001:0.0:1786177354.960021:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177354.960022:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.960022:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177354.960024:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb800 x1872930235607552/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177379 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177354.960027:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177354.960028:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177354.960028:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177354.960029:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177354.960030:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177354.960030:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177354.960031:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177354.960032:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad400. 02000000:00000001:0.0:1786177354.960033:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177354.960034:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177354.960034:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb800. 00000100:00000001:0.0:1786177354.960036:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177354.960037:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.960039:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177354.960043:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177354.960044:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177354.960046:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177354.960047:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177355.468974:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1786177355.468981:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177355.468983:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177355.468988:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:0.0:1786177355.468992:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177355.468995:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177355.468998:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177355.469001:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177355.469004:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177355.852448:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177355.852454:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177355.852460:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177355.852461:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177355.852465:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177355.852466:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177355.863848:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177355.863855:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177355.863861:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177355.863971:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177355.863974:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177355.863976:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177355.863980:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177355.863983:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177355.863987:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177355.863989:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177355.863993:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177355.863995:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177356.493256:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177356.493264:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177356.493267:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177356.493273:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177356.493276:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177356.493279:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177356.493282:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177356.493285:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177356.493288:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177356.876486:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177356.876493:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177356.876500:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177356.876501:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177356.876505:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177356.876506:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177356.876974:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177356.876979:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177356.876982:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177356.877649:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177356.877654:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177356.877656:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177356.877660:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177356.877664:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177356.877668:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177356.877670:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177356.877673:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177356.877677:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177357.516564:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177357.516574:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177357.516577:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177357.516583:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:3.0:1786177357.516586:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177357.516590:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177357.516592:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177357.516594:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177357.516597:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177357.900472:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177357.900479:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177357.900482:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177357.900487:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177357.900491:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177357.900495:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177357.900497:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177357.900499:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177357.900502:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177357.901435:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177357.901438:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177357.901442:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177357.901443:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177357.901447:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177357.901448:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177357.901512:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177357.901516:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177357.901519:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177358.541644:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177358.541652:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177358.541654:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177358.541659:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177358.541790:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177358.541793:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177358.541796:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177358.541799:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177358.541802:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177358.860461:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306705664, at 4306705728 00000100:00000040:1.0:1786177358.860463:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306705664, at 4306705728 00000100:00000010:1.0:1786177358.860477:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5a80. 00000100:00000010:0.0:1786177358.860479:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5060. 00010000:00000001:0.0:1786177358.924489:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177358.924496:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177358.924502:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177358.924503:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177358.924508:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177358.924510:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177358.924566:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177358.924570:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177358.924573:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177358.930121:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177358.930129:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177358.930131:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177358.930136:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177358.930139:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177358.930142:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177358.930145:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177358.930147:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177358.930151:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177359.564466:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177359.564474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177359.564476:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177359.564481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177359.564485:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177359.564489:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177359.564491:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177359.564494:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177359.564497:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177359.948634:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177359.948641:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177359.948646:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177359.948648:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177359.948651:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177359.948652:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177359.948834:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177359.948837:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177359.948840:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177359.948894:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177359.948898:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177359.948901:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177359.948905:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177359.948908:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177359.948911:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177359.948914:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177359.948917:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177359.948920:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177360.076459:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177360.076468:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177360.076471:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177360.076474:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177360.076478:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177360.076485:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12034 00000400:00000200:1.0:1786177360.076493:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177360.076500:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177360.076503:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177360.076506:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076510:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177360.076512:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076514:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177360.076518:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177360.076520:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177360.076522:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177360.076524:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177360.076526:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177360.076527:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177360.076529:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076530:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177360.076532:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076533:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177360.076536:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177360.076538:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076540:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177360.076542:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177360.076544:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177360.076546:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.076547:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177360.076549:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177360.076556:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb100. 00000020:00000040:1.0:1786177360.076559:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177360.076562:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177360.076563:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.076565:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177360.076568:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177360.076570:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177360.076576:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad000. 00000100:00000001:1.0:1786177360.076580:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177360.076587:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177360.076594:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177360.076597:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177360.076600:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f57195c8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177360.076621:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177360.076623:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177360.076625:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177360.076626:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177360.076627:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177360.076630:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177360.076631:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12034 00000400:00000200:1.0:1786177360.076634:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177360.076636:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177360.076638:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177360.076640:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076641:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177360.076642:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076643:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177360.076644:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177360.076645:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177360.076646:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177360.076647:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177360.076648:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177360.076650:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177360.076651:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076652:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177360.076653:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076654:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177360.076655:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:1.0:1786177360.076657:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076658:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177360.076659:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177360.076663:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177360.076664:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.076665:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177360.076668:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177360.076670:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.076671:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177360.076672:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328ba680. 00000020:00000040:1.0:1786177360.076674:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177360.076675:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177360.076676:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.076677:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177360.076678:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177360.076679:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177360.076681:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261afc00. 00000100:00000001:1.0:1786177360.076683:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177360.076686:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177360.076690:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177360.076691:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177360.076693:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000497a117c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177360.076698:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177360.076700:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177360.076702:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177360.076702:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177360.076703:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177360.076705:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177360.076706:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12034 00000400:00000200:1.0:1786177360.076708:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177360.076710:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177360.076711:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177360.076713:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076714:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177360.076715:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076716:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177360.076717:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177360.076717:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177360.076718:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177360.076719:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177360.076720:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177360.076721:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177360.076723:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076725:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177360.076726:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076727:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177360.076728:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:1.0:1786177360.076730:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076731:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177360.076732:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177360.076733:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177360.076734:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.076735:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177360.076736:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177360.076747:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.076748:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177360.076750:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb480. 00000020:00000040:1.0:1786177360.076752:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177360.076753:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177360.076753:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.076754:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177360.076755:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177360.076756:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177360.076758:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af400. 00000100:00000001:1.0:1786177360.076829:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177360.076834:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177360.076841:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177360.076843:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177360.076847:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b192a357] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177360.076869:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177360.076871:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177360.076875:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177360.076876:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177360.076877:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177360.076880:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177360.076882:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12034 00000400:00000200:1.0:1786177360.076886:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177360.076888:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177360.076890:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177360.076892:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076894:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177360.076896:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076898:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177360.076900:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177360.076901:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177360.076903:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177360.076905:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177360.076907:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177360.076909:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:1.0:1786177360.076911:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076913:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177360.076915:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076917:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177360.076919:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:1.0:1786177360.076922:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177360.076924:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177360.076925:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177360.076928:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177360.076930:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.076932:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177360.076934:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177360.076937:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.076938:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177360.076942:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9f80. 00000020:00000040:1.0:1786177360.076944:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177360.076946:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177360.076947:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.076949:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177360.076950:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177360.076952:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177360.076955:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261adc00. 00000100:00000001:1.0:1786177360.076961:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177360.076966:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177360.076972:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177360.076974:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177360.076977:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4f3e5a2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177360.076984:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177360.076986:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177360.076989:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177360.076990:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177360.076992:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177360.076994:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177360.076998:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12034 00000400:00000200:1.0:1786177360.077001:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177360.077004:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177360.077005:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177360.077008:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.077009:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177360.077011:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.077013:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177360.077015:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177360.077016:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177360.077018:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177360.077019:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:1.0:1786177360.077021:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:1.0:1786177360.077022:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:1.0:1786177360.077025:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.077026:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177360.077028:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177360.077029:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177360.077031:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:1.0:1786177360.077033:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177360.077035:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177360.077037:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177360.077039:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177360.077040:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.077042:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177360.077043:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177360.077045:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.077047:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177360.077049:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9c00. 00000020:00000040:1.0:1786177360.077051:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177360.077053:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177360.077054:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.077055:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177360.077056:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177360.077058:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177360.077060:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261acc00. 00000100:00000001:1.0:1786177360.077062:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177360.077066:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177360.077077:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177360.077078:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177360.077082:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3ea07ed] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177360.077088:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177360.077092:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12044) 00000100:00000001:0.0:1786177360.077848:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177360.077853:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177360.077855:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177360.077863:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.077872:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177360.077874:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177360.077878:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177360.077881:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177360.077883:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177360.077887:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235607680:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177360.077891:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177360.077895:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177360.077898:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177360.077900:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177360.077905:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177360.077907:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177360.077912:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177360.077916:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177360.077923:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177360.077930:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235607680, portal 25 00000100:00000001:0.0:1786177360.077932:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177360.077934:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487994624 : -110371221556992 : ffff9b9e328bb100) 00000100:00000040:0.0:1786177360.077940:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.077947:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177360.077950:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177360.077953:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177360.077955:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235607680, offset 0 00000400:00000200:0.0:1786177360.077962:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177360.077970:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177360.077975:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177360.077980:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177360.077983:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177360.077987:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177360.077989:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177360.077993:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426814:426814:256:4294967295] 192.168.202.132@tcp LPNI seq info [426776:426776:8:4294967295] 00000400:00000200:0.0:1786177360.077997:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177360.078006:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177360.078074:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177360.078080:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177360.078086:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177360.078090:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1786177360.078095:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177360.078098:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1786177360.078103:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177360.078105:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177360.078111:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177360.078116:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177360.078121:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177360.078123:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177360.078127:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177360.078131:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078141:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177360.078144:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078156:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177360.078158:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177360.078164:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177360.078167:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177360.078170:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306710785 00000100:00000010:0.0:1786177360.078175:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5060. 00000100:00000001:0.0:1786177360.078181:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177360.078187:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306710785 00000100:00000010:0.0:1786177360.078190:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d55a0. 00000100:00000001:0.0:1786177360.078193:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177360.078196:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177360.078199:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177360.078201:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177360.078205:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177360/real 1786177360] req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177360.078214:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177360.078216:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177360.078218:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177360.078220:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177360.078222:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177360.078224:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177360.078228:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177360.078235:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078242:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177360.078243:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.078245:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177360.078247:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177360.078251:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078258:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177360.078263:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078269:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177360.078272:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.078274:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177360.078275:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.078277:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177360.078279:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177360.078282:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177360.078286:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177360.078288:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177360.078293:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177360.078301:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235607680:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177360.078305:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177360.078308:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078315:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177360.078317:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177360.078319:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177360.078321:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177360.078323:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177360.078326:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607808:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177360.078329:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177360.078331:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177360.078332:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177360.078335:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177360.078338:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177360.078340:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177360.078344:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177360.078346:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177360.078349:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177360.078352:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235607808, portal 10 00000100:00000001:0.0:1786177360.078354:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177360.078355:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487991936 : -110371221559680 : ffff9b9e328ba680) 00000100:00000040:0.0:1786177360.078387:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078393:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177360.078395:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177360.078397:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177360.078400:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235607808, offset 0 00000400:00000200:0.0:1786177360.078403:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177360.078407:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177360.078410:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177360.078412:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177360.078415:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177360.078416:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177360.078418:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177360.078421:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426815:426815:256:4294967295] 192.168.202.132@tcp LPNI seq info [426777:426777:8:4294967295] 00000400:00000200:0.0:1786177360.078424:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177360.078430:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177360.078434:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177360.078437:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177360.078440:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177360.078442:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1786177360.078446:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177360.078448:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1786177360.078452:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177360.078453:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177360.078456:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177360.078459:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177360.078461:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177360.078463:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177360.078465:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177360.078468:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078474:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177360.078477:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078483:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177360.078484:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177360.078487:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177360.078491:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177360.078494:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177360.078496:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306706949 00000100:00000001:0.0:1786177360.078498:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306710785 00000100:00000001:0.0:1786177360.078500:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306706949 00000100:00000001:0.0:1786177360.078502:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306710785 00000100:00000001:0.0:1786177360.078504:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177360.078506:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177360.078507:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177360.078511:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177360/real 1786177360] req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177360.078517:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177360.078518:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177360.078520:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177360.078522:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177360.078523:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177360.078525:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177360.078528:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177360.078535:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078541:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177360.078542:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.078544:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177360.078545:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177360.078549:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078555:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177360.078558:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078565:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177360.078567:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.078569:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177360.078570:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.078571:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177360.078574:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177360.078576:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177360.078579:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177360.078581:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177360.078585:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177360.078592:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607808:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177360.078595:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177360.078598:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078609:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177360.078610:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177360.078612:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177360.078614:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177360.078616:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177360.078619:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607936:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177360.078622:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177360.078624:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177360.078625:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177360.078627:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177360.078630:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177360.078632:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177360.078635:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177360.078638:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177360.078641:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177360.078643:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235607936, portal 10 00000100:00000001:0.0:1786177360.078645:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177360.078647:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487995520 : -110371221556096 : ffff9b9e328bb480) 00000100:00000040:0.0:1786177360.078652:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078658:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177360.078660:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177360.078662:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177360.078664:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235607936, offset 0 00000400:00000200:0.0:1786177360.078668:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177360.078671:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177360.078675:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177360.078677:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177360.078680:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177360.078682:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177360.078684:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177360.078687:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426816:426816:256:4294967295] 192.168.202.132@tcp LPNI seq info [426778:426778:8:4294967295] 00000400:00000200:0.0:1786177360.078692:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177360.078698:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177360.078703:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177360.078706:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177360.078710:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177360.078712:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1786177360.078717:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177360.078719:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1786177360.078723:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177360.078725:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177360.078728:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177360.078730:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177360.078733:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177360.078735:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177360.078738:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177360.078741:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.078749:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177360.078752:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.080856:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177360.080864:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177360.080874:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177360.080878:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177360.080885:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177360.080887:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306706951 00000100:00000001:0.0:1786177360.080891:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306710785 00000100:00000001:0.0:1786177360.080894:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306706951 00000100:00000001:0.0:1786177360.080896:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306710785 00000100:00000001:0.0:1786177360.080899:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177360.080902:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177360.080904:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177360.080913:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177360/real 1786177360] req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177360.080925:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177360.080927:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177360.080929:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177360.080933:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177360.080935:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177360.080937:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177360.080941:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177360.080950:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.080957:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177360.080959:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.080961:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177360.080963:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177360.080966:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.080973:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177360.080977:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.080984:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177360.080986:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.080988:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177360.080990:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.080992:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177360.080994:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177360.080998:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177360.081003:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177360.081005:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177360.081009:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177360.081918:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235607936:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177360.081923:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177360.081927:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.081934:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177360.081936:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177360.081940:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177360.081943:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177360.081945:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177360.081948:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608064:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177360.081952:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177360.081955:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177360.081957:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177360.081960:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177360.081967:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177360.081969:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177360.081974:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177360.081977:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177360.081981:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177360.081988:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235608064, portal 4 00000100:00000001:0.0:1786177360.081990:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177360.081992:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487990144 : -110371221561472 : ffff9b9e328b9f80) 00000100:00000040:0.0:1786177360.081998:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082005:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177360.082008:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177360.082108:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177360.082113:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235608064, offset 0 00000400:00000200:0.0:1786177360.082119:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177360.082127:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177360.082133:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177360.082138:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177360.082142:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177360.082146:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177360.082149:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177360.082152:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426817:426817:256:4294967295] 192.168.202.132@tcp LPNI seq info [426779:426779:8:4294967295] 00000400:00000200:0.0:1786177360.082157:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177360.082165:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177360.082170:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177360.082175:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177360.082182:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177360.082185:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1786177360.082190:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177360.082194:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1786177360.082197:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177360.082200:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177360.082204:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177360.082209:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177360.082213:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177360.082216:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177360.082218:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177360.082222:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082231:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177360.082234:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082241:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177360.082242:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177360.082246:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177360.082250:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177360.082253:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177360.082255:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306706952 00000100:00000001:0.0:1786177360.082258:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306710785 00000100:00000001:0.0:1786177360.082260:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306706952 00000100:00000001:0.0:1786177360.082263:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306710785 00000100:00000001:0.0:1786177360.082265:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177360.082267:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177360.082269:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177360.082272:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177360/real 1786177360] req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177360.082280:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177360.082281:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177360.082282:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177360.082285:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177360.082286:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177360.082288:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177360.082291:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177360.082299:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082305:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177360.082307:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.082309:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177360.082310:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177360.082313:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082319:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177360.082323:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082330:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177360.082333:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.082334:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177360.082335:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.082337:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177360.082339:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177360.082342:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177360.082345:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177360.082347:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177360.082351:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177360.082374:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608064:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177360.082378:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177360.082382:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082388:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177360.082390:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177360.082392:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177360.082393:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177360.082395:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177360.082398:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608192:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177360.082401:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177360.082403:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177360.082404:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177360.082406:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177360.082409:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177360.082411:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177360.082414:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177360.082417:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177360.082420:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177360.082423:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235608192, portal 4 00000100:00000001:0.0:1786177360.082425:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177360.082426:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487989248 : -110371221562368 : ffff9b9e328b9c00) 00000100:00000040:0.0:1786177360.082431:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082438:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177360.082441:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177360.082443:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177360.082445:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235608192, offset 0 00000400:00000200:0.0:1786177360.082449:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177360.082453:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177360.082457:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177360.082459:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177360.082462:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177360.082464:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177360.082467:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177360.082470:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426818:426818:256:4294967295] 192.168.202.132@tcp LPNI seq info [426780:426780:8:4294967295] 00000400:00000200:0.0:1786177360.082474:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177360.082481:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177360.082485:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177360.082488:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177360.082492:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177360.082495:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1786177360.082499:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177360.082502:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 7 secs later) 00000800:00000100:0.0:1786177360.082505:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177360.082507:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177360.082510:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177360.082514:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177360.082516:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177360.082518:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177360.082521:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177360.082525:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082532:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177360.082535:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177385 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082541:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177360.082543:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177360.082545:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177360.082548:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177360.082550:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177360.082551:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306706955 00000100:00000001:0.0:1786177360.082554:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306710785 00000100:00000001:0.0:1786177360.082556:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306706955 00000100:00000001:0.0:1786177360.082558:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306710785 00000100:00000001:0.0:1786177360.082559:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177360.082561:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177360.082563:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177360.082566:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177360/real 1786177360] req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177360.082573:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177360.082574:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177360.082576:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177360.082578:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177360.082580:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177360.082582:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177360.082585:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177360.082593:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082604:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177360.082606:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.082607:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177360.082609:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177360.082612:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082617:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177360.082620:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082626:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177360.082628:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.082629:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177360.082630:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.082632:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177360.082633:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177360.082635:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177360.082638:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177360.082640:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177360.082643:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177360.082650:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608192:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177360.082653:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.082656:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177360.082660:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb100 x1872930235607680/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082666:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177360.082668:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177360.082671:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177360.082675:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177360.082677:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177360.082679:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177360.082681:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177360.082684:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad000. 02000000:00000001:0.0:1786177360.082687:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177360.082689:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177360.082691:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb100. 00000100:00000001:0.0:1786177360.082695:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177360.082697:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.082698:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177360.082702:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328ba680 x1872930235607808/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082708:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177360.082709:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177360.082711:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177360.082713:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177360.082714:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177360.082716:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177360.082717:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177360.082719:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:0.0:1786177360.082721:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177360.082722:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177360.082723:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328ba680. 00000100:00000001:0.0:1786177360.082726:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177360.082727:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.082728:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177360.082731:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb480 x1872930235607936/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.082737:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177360.082739:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177360.082740:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177360.082742:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177360.082743:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177360.082744:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177360.082747:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177360.082748:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af400. 02000000:00000001:0.0:1786177360.082750:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177360.082751:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177360.082752:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb480. 00000100:00000001:0.0:1786177360.082754:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177360.082755:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.082756:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177360.085143:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9f80 x1872930235608064/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.085155:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177360.085157:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177360.085161:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177360.085165:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177360.085167:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177360.085168:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177360.085171:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177360.085173:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261adc00. 02000000:00000001:0.0:1786177360.085175:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177360.085177:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177360.085178:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9f80. 00000100:00000001:0.0:1786177360.085182:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177360.085183:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.085186:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177360.085190:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9c00 x1872930235608192/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177385 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177360.085197:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177360.085199:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177360.085200:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177360.085202:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177360.085203:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177360.085204:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177360.085206:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177360.085207:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:0.0:1786177360.085209:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177360.085211:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177360.085213:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9c00. 00000100:00000001:0.0:1786177360.085215:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177360.085217:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.085219:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177360.085227:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177360.085229:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177360.085231:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177360.085233:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.589178:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177360.589187:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177360.589190:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177360.589197:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177360.589201:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177360.589206:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177360.589210:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177360.589213:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177360.589216:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177360.972558:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177360.972567:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177360.972569:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177360.972575:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177360.972580:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177360.972585:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177360.972588:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177360.972591:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177360.972594:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177360.972710:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177360.972714:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177360.972718:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177360.972721:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177360.972724:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177360.972728:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177360.972731:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177360.972734:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177360.972736:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177360.972738:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177360.972742:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177360.972743:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177360.972828:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177360.972832:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177360.972835:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177361.612890:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177361.612897:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177361.612900:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177361.612906:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177361.612910:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177361.612914:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177361.612917:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177361.612920:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177361.612922:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177361.996466:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177361.996474:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177361.996477:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177361.996483:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177361.996486:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177361.996496:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177361.996499:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177361.996501:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177361.996504:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177361.996868:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177361.996875:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177361.996881:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177361.996882:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177361.996886:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177361.996887:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177361.997013:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177361.997018:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177361.997021:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177362.640151:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177362.640162:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177362.640165:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177362.640171:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177362.640176:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177362.640181:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177362.640184:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177362.640187:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177362.640191:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177363.020534:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177363.020546:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177363.020549:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177363.020556:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177363.020561:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177363.020567:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177363.020571:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177363.020574:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177363.020578:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177363.020719:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177363.020722:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177363.020727:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177363.020728:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177363.020765:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177363.020767:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177363.020863:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177363.020867:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177363.020871:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177363.660614:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177363.660624:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177363.660627:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177363.660634:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177363.660639:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177363.660644:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177363.660648:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177363.660652:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177363.660656:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177363.980525:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306710785, at 4306710848 00000100:00000010:0.0:1786177363.980546:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5060. 00000100:00000040:1.0:1786177363.980793:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306710785, at 4306710848 00000100:00000010:1.0:1786177363.980800:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d55a0. 00000100:00000001:1.0:1786177364.044516:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177364.044526:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177364.044528:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177364.044535:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177364.044539:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177364.044544:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177364.044547:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177364.044550:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177364.044554:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177364.044665:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177364.044671:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177364.044673:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177364.044678:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177364.044681:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177364.044684:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177364.044686:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177364.044688:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177364.044692:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177364.044694:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177364.044697:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177364.044698:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177364.044699:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177364.044701:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177364.044703:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177364.044704:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177364.044705:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177364.044707:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177364.046559:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177364.046565:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177364.046570:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177364.684505:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177364.684514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177364.684516:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177364.684523:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177364.684528:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177364.684532:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177364.684535:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177364.684539:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177364.684543:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177365.068728:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177365.068737:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177365.068739:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177365.068746:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177365.068755:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177365.068760:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177365.068764:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177365.068767:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177365.068771:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177365.069153:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177365.069157:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177365.069162:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177365.069164:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177365.069167:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177365.069169:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177365.069752:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177365.069757:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177365.069760:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177365.197438:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177365.197452:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177365.197455:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177365.197460:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177365.197466:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177365.197552:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12039 00000400:00000200:2.0:1786177365.197564:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177365.197572:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177365.197575:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177365.197578:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197582:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177365.197585:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197588:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177365.197593:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177365.197595:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177365.197598:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177365.197601:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177365.197603:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177365.197606:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177365.197609:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197611:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177365.197613:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197615:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177365.197618:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177365.197621:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197623:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177365.197626:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177365.197630:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177365.197633:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.197636:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177365.197638:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177365.197650:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 00000020:00000040:2.0:1786177365.197653:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177365.197657:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177365.197658:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.197661:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177365.197665:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177365.197669:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177365.197678:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:2.0:1786177365.197685:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177365.197696:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177365.197706:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177365.197722:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177365.197727:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177365.197751:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177365.197753:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177365.197756:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177365.197757:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177365.197759:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177365.197761:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177365.197763:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12039 00000400:00000200:2.0:1786177365.197767:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177365.197770:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177365.197772:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177365.197775:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197778:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177365.197779:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197781:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177365.197784:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177365.197785:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177365.197787:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177365.197789:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177365.197791:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177365.197793:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177365.197796:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197797:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177365.197799:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197801:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177365.197804:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177365.197806:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197809:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177365.197811:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177365.197818:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177365.197819:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.197821:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177365.197825:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177365.197827:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.197829:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177365.197832:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786177365.197834:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177365.197836:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177365.197836:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.197838:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177365.197839:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177365.197840:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177365.197843:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab400. 00000100:00000001:2.0:1786177365.197846:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177365.197849:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177365.197854:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177365.197856:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177365.197858:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177365.197863:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177365.197865:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177365.197867:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177365.197868:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177365.197869:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177365.197871:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177365.197873:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12039 00000400:00000200:2.0:1786177365.197875:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177365.197877:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177365.197878:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177365.197879:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197881:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177365.197882:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197883:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177365.197885:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177365.197886:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177365.197887:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177365.197888:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177365.197890:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177365.197891:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177365.197893:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197893:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177365.197894:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197895:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177365.197897:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177365.197898:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197899:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177365.197900:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177365.197901:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177365.197902:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.197903:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177365.197905:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177365.197906:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.197907:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177365.197908:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786177365.197909:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177365.197911:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177365.197911:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.197912:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177365.197913:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177365.197914:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177365.197916:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786177365.197918:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177365.197920:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177365.197924:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177365.197926:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177365.197930:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177365.197938:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177365.197940:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177365.197944:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177365.197946:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177365.197947:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177365.197950:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177365.197953:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12039 00000400:00000200:2.0:1786177365.197956:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177365.197959:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177365.197960:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177365.197963:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197965:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177365.197966:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197968:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177365.197970:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177365.197971:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177365.197973:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177365.197975:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177365.197977:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177365.197979:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177365.197982:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197983:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177365.197985:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197987:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177365.197990:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177365.197992:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177365.197995:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177365.197997:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177365.198000:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177365.198001:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.198003:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177365.198006:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177365.198007:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.198009:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177365.198012:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000020:00000040:2.0:1786177365.198014:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177365.198016:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177365.198017:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.198019:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177365.198020:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177365.198022:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177365.198027:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:2.0:1786177365.198030:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177365.198033:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177365.198040:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177365.198041:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177365.198044:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177365.198051:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177365.198054:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177365.198057:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177365.198058:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177365.198059:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177365.198061:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177365.198063:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12039 00000400:00000200:2.0:1786177365.198067:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177365.198070:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177365.198072:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177365.198074:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.198076:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177365.198077:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.198079:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177365.198081:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177365.198082:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177365.198083:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177365.198085:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177365.198086:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177365.198088:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177365.198091:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.198092:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177365.198094:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177365.198096:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177365.198097:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177365.198100:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177365.198101:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177365.198103:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177365.198105:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177365.198106:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.198108:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177365.198110:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177365.198112:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.198113:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177365.198116:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737480. 00000020:00000040:2.0:1786177365.198118:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177365.198120:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177365.198121:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.198123:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177365.198124:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177365.198125:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177365.198128:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:2.0:1786177365.198132:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177365.198136:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177365.198143:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177365.198145:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177365.198148:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046bba14a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177365.198155:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177365.198160:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12049) 00000100:00000001:0.0:1786177365.198421:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177365.198426:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177365.198429:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177365.198437:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.198445:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177365.198448:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177365.198451:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177365.198455:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177365.198457:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177365.198461:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235608320:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177365.198466:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177365.198471:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177365.198707:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177365.198720:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177365.198727:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177365.198729:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177365.198736:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177365.198745:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177365.198754:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177365.198764:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235608320, portal 25 00000100:00000001:0.0:1786177365.198766:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177365.198768:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469630848 : -110371239920768 : ffff9b9e31737b80) 00000100:00000040:0.0:1786177365.198774:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.198782:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177365.198784:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177365.198787:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177365.198790:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235608320, offset 0 00000400:00000200:0.0:1786177365.198798:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177365.198807:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177365.198814:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177365.198820:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177365.198825:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177365.198829:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177365.198831:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177365.198834:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426819:426819:256:4294967295] 192.168.202.132@tcp LPNI seq info [426781:426781:8:4294967295] 00000400:00000200:0.0:1786177365.198837:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177365.198846:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177365.198852:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177365.198857:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177365.198863:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177365.198866:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1786177365.198870:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177365.198873:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1786177365.198876:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177365.198878:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177365.198884:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177365.198889:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177365.198893:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177365.198895:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177365.198899:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177365.198902:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.198910:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177365.198912:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.198917:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177365.198918:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177365.198924:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177365.198927:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177365.198929:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306715906 00000100:00000010:0.0:1786177365.198932:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5060. 00000100:00000001:0.0:1786177365.198939:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177365.198944:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306715906 00000100:00000010:0.0:1786177365.198946:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5ea0. 00000100:00000001:0.0:1786177365.198949:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177365.198951:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177365.198953:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177365.198955:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177365.198958:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177365/real 1786177365] req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177365.198965:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177365.198967:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177365.198969:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177365.198971:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177365.198973:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177365.198975:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177365.198979:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177365.198993:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199000:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177365.199002:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.199004:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177365.199006:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177365.199010:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199016:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177365.199019:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199024:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177365.199025:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.199026:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177365.199027:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.199029:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177365.199030:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177365.199032:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177365.199035:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177365.199036:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177365.199038:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177365.199043:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235608320:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177365.199047:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177365.199049:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199053:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177365.199055:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177365.199057:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177365.199058:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177365.199059:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177365.199061:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608448:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177365.199063:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177365.199065:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177365.199066:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177365.199067:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177365.199069:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177365.199071:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177365.199073:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177365.199075:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177365.199078:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177365.199080:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235608448, portal 10 00000100:00000001:0.0:1786177365.199081:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177365.199082:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620992 : -110371239930624 : ffff9b9e31735500) 00000100:00000040:0.0:1786177365.199087:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199092:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177365.199093:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177365.199094:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177365.199096:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235608448, offset 0 00000400:00000200:0.0:1786177365.199098:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177365.199100:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177365.199103:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177365.199105:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177365.199106:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177365.199108:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177365.199110:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177365.199112:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426820:426820:256:4294967295] 192.168.202.132@tcp LPNI seq info [426782:426782:8:4294967295] 00000400:00000200:0.0:1786177365.199114:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177365.199118:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177365.199121:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177365.199123:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177365.199126:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177365.199127:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1786177365.199130:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177365.199133:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1786177365.199138:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177365.199140:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177365.199142:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177365.199146:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177365.199148:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177365.199150:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177365.199152:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177365.199155:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199162:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177365.199165:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199171:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177365.199173:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177365.199176:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177365.199180:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177365.199184:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177365.199186:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306712069 00000100:00000001:0.0:1786177365.199188:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306715906 00000100:00000001:0.0:1786177365.199190:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306712069 00000100:00000001:0.0:1786177365.199191:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306715906 00000100:00000001:0.0:1786177365.199193:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177365.199194:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177365.199196:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177365.199200:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177365/real 1786177365] req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177365.199209:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177365.199210:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177365.199211:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177365.199214:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177365.199216:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177365.199218:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177365.199221:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177365.199229:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199236:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177365.199237:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.199239:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177365.199240:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177365.199244:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199250:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177365.199253:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199260:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177365.199262:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.199264:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177365.199265:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.199267:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177365.199270:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177365.199273:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177365.199276:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177365.199279:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177365.199283:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177365.199289:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608448:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177365.199293:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177365.199296:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199302:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177365.199304:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177365.199306:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177365.199308:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177365.199310:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177365.199312:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608576:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177365.199316:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177365.199318:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177365.199320:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177365.199321:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177365.199325:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177365.199327:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177365.199331:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177365.199333:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177365.199336:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177365.199339:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235608576, portal 10 00000100:00000001:0.0:1786177365.199341:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177365.199342:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469622784 : -110371239928832 : ffff9b9e31735c00) 00000100:00000040:0.0:1786177365.199348:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.199354:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177365.199409:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177365.199412:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177365.199415:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235608576, offset 0 00000400:00000200:0.0:1786177365.199420:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177365.199424:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177365.199428:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177365.199431:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177365.199433:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177365.199436:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177365.199438:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177365.199440:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426821:426821:256:4294967295] 192.168.202.132@tcp LPNI seq info [426783:426783:8:4294967295] 00000400:00000200:0.0:1786177365.199444:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177365.199450:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177365.199455:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177365.199458:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177365.199462:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177365.199465:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1786177365.199469:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177365.199471:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1786177365.200079:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177365.200082:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177365.200086:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177365.200089:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177365.200093:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177365.200095:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177365.200097:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177365.200101:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200110:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177365.200113:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200120:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177365.200121:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177365.200125:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177365.200128:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177365.200131:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177365.200135:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306712070 00000100:00000001:0.0:1786177365.200138:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306715906 00000100:00000001:0.0:1786177365.200140:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306712070 00000100:00000001:0.0:1786177365.200142:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306715906 00000100:00000001:0.0:1786177365.200144:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177365.200147:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177365.200148:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177365.200152:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177365/real 1786177365] req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177365.200160:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177365.200161:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177365.200163:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177365.200165:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177365.200167:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177365.200169:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177365.200172:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177365.200180:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200187:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177365.200188:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.200190:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177365.200192:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177365.200195:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200202:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177365.200206:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200212:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177365.200215:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.200217:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177365.200218:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.200219:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177365.200221:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177365.200224:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177365.200228:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177365.200230:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177365.200233:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177365.200241:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608576:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177365.200245:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177365.200248:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200255:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177365.200256:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177365.200258:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177365.200260:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177365.200262:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177365.200264:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608704:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177365.200267:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177365.200269:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177365.200271:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177365.200273:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177365.200277:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177365.200279:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177365.200283:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177365.200287:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177365.200290:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177365.200293:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235608704, portal 4 00000100:00000001:0.0:1786177365.200296:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177365.200297:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469621888 : -110371239929728 : ffff9b9e31735880) 00000100:00000040:0.0:1786177365.200302:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200308:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177365.200311:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177365.200313:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177365.200315:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235608704, offset 0 00000400:00000200:0.0:1786177365.200319:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177365.200323:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177365.200327:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177365.200330:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177365.200333:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177365.200335:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177365.200337:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177365.200340:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426822:426822:256:4294967295] 192.168.202.132@tcp LPNI seq info [426784:426784:8:4294967295] 00000400:00000200:0.0:1786177365.200344:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177365.200350:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177365.200354:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177365.200450:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177365.200455:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177365.200458:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1786177365.200462:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177365.200465:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1786177365.200468:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177365.200470:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177365.200555:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177365.200560:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177365.200564:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177365.200566:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177365.200569:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177365.200573:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200586:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177365.200590:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200597:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177365.200598:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177365.200601:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177365.200604:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177365.200607:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177365.200609:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306712071 00000100:00000001:0.0:1786177365.200612:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306715906 00000100:00000001:0.0:1786177365.200614:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306712071 00000100:00000001:0.0:1786177365.200616:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306715906 00000100:00000001:0.0:1786177365.200618:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177365.200620:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177365.200622:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177365.200626:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177365/real 1786177365] req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177365.200634:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177365.200635:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177365.200637:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177365.200639:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177365.200641:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177365.200643:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177365.200646:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177365.200654:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200660:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177365.200662:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.200664:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177365.200665:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177365.200668:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200675:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177365.200678:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200685:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177365.200687:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.200689:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177365.200691:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.200692:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177365.200694:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177365.200696:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177365.200700:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177365.200702:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177365.200705:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177365.200726:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608704:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177365.200730:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177365.200734:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200740:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177365.200742:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177365.200744:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177365.200746:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177365.200748:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177365.200751:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608832:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177365.200754:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177365.200756:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177365.200758:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177365.200760:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177365.200763:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177365.200765:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177365.200769:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177365.200771:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177365.200774:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177365.200777:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235608832, portal 4 00000100:00000001:0.0:1786177365.200779:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177365.200780:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469629056 : -110371239922560 : ffff9b9e31737480) 00000100:00000040:0.0:1786177365.200786:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200792:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177365.200794:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177365.200796:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177365.200799:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235608832, offset 0 00000400:00000200:0.0:1786177365.200803:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177365.200806:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177365.200810:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177365.200813:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177365.200815:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177365.200818:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177365.200820:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177365.200823:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426823:426823:256:4294967295] 192.168.202.132@tcp LPNI seq info [426785:426785:8:4294967295] 00000400:00000200:0.0:1786177365.200827:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177365.200832:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177365.200837:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177365.200840:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177365.200844:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177365.200847:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1786177365.200851:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177365.200854:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1786177365.200857:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177365.200859:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:0.0:1786177365.200862:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177365.200865:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177365.200868:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:0.0:1786177365.200869:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:0.0:1786177365.200871:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177365.200875:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200882:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177365.200885:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177390 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200891:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177365.200893:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177365.200896:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177365.200899:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177365.200902:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177365.200904:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306712073 00000100:00000001:0.0:1786177365.200907:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306715906 00000100:00000001:0.0:1786177365.200909:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306712073 00000100:00000001:0.0:1786177365.200911:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306715906 00000100:00000001:0.0:1786177365.200913:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177365.200915:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177365.200917:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177365.200920:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177365/real 1786177365] req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177365.200928:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177365.200930:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177365.200931:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000400:00000200:0.0:1786177365.200933:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:0.0:1786177365.200935:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000100:00000001:0.0:1786177365.200937:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177365.200939:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177365.200946:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200952:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177365.200953:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.200954:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177365.200956:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177365.200959:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200965:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177365.200968:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.200975:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177365.200977:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.200979:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177365.200980:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.200982:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177365.200983:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177365.200986:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177365.200989:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177365.200991:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177365.200995:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177365.201009:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235608832:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177365.201013:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.201016:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177365.201019:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737b80 x1872930235608320/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.201026:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177365.201029:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177365.201031:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177365.201034:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177365.201036:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177365.201038:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177365.201040:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177365.201043:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:0.0:1786177365.201047:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177365.201049:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177365.201051:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737b80. 00000100:00000001:0.0:1786177365.201055:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177365.201057:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.201058:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177365.201062:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735500 x1872930235608448/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.201068:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177365.201069:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177365.201070:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177365.201072:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177365.201073:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177365.201075:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177365.201076:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177365.201078:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:0.0:1786177365.201080:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177365.201081:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177365.201082:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735500. 00000100:00000001:0.0:1786177365.201084:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177365.201085:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.201087:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177365.201089:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735c00 x1872930235608576/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.201094:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177365.201095:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177365.201096:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177365.201098:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177365.201099:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177365.201100:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177365.201102:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177365.201102:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:0.0:1786177365.201104:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177365.201105:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177365.201106:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735c00. 00000100:00000001:0.0:1786177365.201107:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177365.201108:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.201110:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177365.201112:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735880 x1872930235608704/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.201118:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177365.201119:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177365.201120:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177365.201122:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177365.201123:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177365.201124:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177365.201126:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177365.201127:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:0.0:1786177365.201129:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177365.201130:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177365.201131:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735880. 00000100:00000001:0.0:1786177365.201132:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177365.201133:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.201134:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177365.201137:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737480 x1872930235608832/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177390 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177365.201143:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177365.201145:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177365.201146:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177365.201148:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177365.201149:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177365.201150:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177365.201152:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177365.201154:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:0.0:1786177365.201156:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177365.201157:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177365.201158:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737480. 00000100:00000001:0.0:1786177365.201160:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177365.201161:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.201164:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177365.201173:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177365.201176:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177365.201178:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177365.201180:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177365.708624:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177365.708635:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177365.708637:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177365.708644:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177365.708648:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177365.708651:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177365.708654:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177365.708657:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177365.708661:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177366.092432:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177366.092440:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177366.092442:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177366.092447:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177366.092451:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177366.092455:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177366.092457:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177366.092460:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177366.092463:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177366.092467:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177366.092473:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177366.092479:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177366.092481:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177366.092485:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177366.092487:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177366.093151:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177366.093156:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177366.093159:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177366.732875:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177366.732885:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177366.732888:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177366.732895:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177366.732898:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177366.732903:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177366.732906:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177366.732909:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177366.732913:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177367.116500:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177367.116508:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177367.116516:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177367.116517:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177367.116521:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177367.116523:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177367.116625:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177367.116630:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177367.116632:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177367.116636:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177367.116640:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177367.116644:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177367.116647:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177367.116650:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177367.116654:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177367.116728:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177367.116732:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177367.116735:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177367.757436:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177367.757446:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177367.757449:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177367.757456:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177367.757460:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177367.757464:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177367.757467:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177367.757471:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177367.757474:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177368.140491:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177368.140500:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177368.140501:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177368.140502:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177368.140507:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177368.140509:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177368.140514:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177368.140515:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177368.140517:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177368.140518:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177368.140525:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177368.140526:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177368.140535:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177368.140538:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177368.140542:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177368.140629:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177368.140634:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177368.140637:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177368.780742:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177368.780751:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177368.780753:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177368.780758:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177368.780762:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177368.780766:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177368.780768:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177368.780769:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177368.780771:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177369.100543:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306715906, at 4306715968 00000100:00000010:0.0:1786177369.100570:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5060. 00000100:00000040:1.0:1786177369.100663:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306715906, at 4306715968 00000100:00000010:1.0:1786177369.100668:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5ea0. 00000100:00000001:1.0:1786177369.164850:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177369.164861:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177369.164864:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177369.164872:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177369.164876:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177369.164883:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177369.164887:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177369.164890:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177369.164894:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177369.167438:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177369.167449:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177369.167461:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177369.167462:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177369.167467:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177369.167468:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177369.167869:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177369.167873:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177369.167876:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177369.804511:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177369.804523:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177369.804527:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177369.804544:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177369.804551:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177369.804556:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177369.804559:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177369.804562:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177369.804566:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177370.188438:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177370.188445:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177370.188447:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177370.188452:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177370.188455:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177370.188458:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177370.188460:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177370.188463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177370.188465:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177370.188795:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177370.188802:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177370.188809:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177370.188810:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177370.188814:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177370.188815:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177370.188903:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177370.188907:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177370.188911:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177370.316908:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177370.316925:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177370.316928:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177370.316933:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177370.316939:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177370.316948:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12044 00000400:00000200:2.0:1786177370.316959:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177370.316967:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177370.316970:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177370.316974:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.316978:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177370.316980:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.316983:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177370.316988:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177370.316991:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177370.316994:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177370.316998:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177370.317006:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177370.317010:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177370.317013:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.317015:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177370.317017:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.317019:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177370.317022:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177370.317025:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177370.317028:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177370.317031:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177370.317035:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177370.317039:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.317041:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177370.317044:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177370.317058:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736d80. 00000020:00000040:2.0:1786177370.317062:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177370.317067:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177370.317069:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.317072:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177370.317077:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177370.317082:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177370.317091:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:2.0:1786177370.317100:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177370.317111:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177370.317123:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177370.317128:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177370.317133:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009e4f7686] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.317277:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177370.317283:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177370.317286:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177370.317294:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.317304:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177370.317306:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177370.317310:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177370.317313:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177370.317316:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177370.317319:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235608960:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177370.317324:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177370.317329:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177370.317332:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177370.317334:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177370.317340:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177370.317342:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177370.317349:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177370.317354:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e580. 00000400:00000010:0.0:1786177370.317387:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686110. 00000100:00000200:0.0:1786177370.317397:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235608960, portal 25 00000100:00000001:0.0:1786177370.317400:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177370.317403:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469627264 : -110371239924352 : ffff9b9e31736d80) 00000100:00000040:0.0:1786177370.317408:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.317416:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177370.317419:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177370.317421:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686088. 00000100:00000200:0.0:1786177370.317424:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235608960, offset 0 00000400:00000200:0.0:1786177370.317432:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177370.317441:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177370.317448:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177370.317454:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177370.317459:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177370.317463:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177370.317466:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177370.317470:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426824:426824:256:4294967295] 192.168.202.132@tcp LPNI seq info [426786:426786:8:4294967295] 00000400:00000200:0.0:1786177370.317475:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177370.317486:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177370.317492:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177370.317498:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ca00. 00000800:00000200:0.0:1786177370.317507:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177370.317512:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177370.317544:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177370.317549:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177370.317552:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306721024 00000100:00000010:0.0:1786177370.317557:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5060. 00000100:00000001:0.0:1786177370.317565:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177370.317575:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306721024 00000100:00000010:0.0:1786177370.317578:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d56c0. 00000100:00000001:0.0:1786177370.317582:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177370.317586:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177370.317590:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177370.317593:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177370.317597:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177370.317643:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177370.317647:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177370.317650:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177370.317653:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177370.317655:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177370.317658:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177370.317673:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12044 00000400:00000200:2.0:1786177370.317677:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177370.317681:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177370.317684:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177370.317686:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.317689:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177370.317691:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.317693:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177370.317696:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177370.317697:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177370.317699:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177370.317702:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177370.317704:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177370.317706:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177370.317709:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.317711:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177370.317713:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.317715:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177370.317717:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177370.317719:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177370.317722:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177370.317724:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177370.317731:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177370.317733:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.317735:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177370.317739:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177370.317742:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.317745:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177370.317749:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734700. 00000020:00000040:2.0:1786177370.317751:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177370.317754:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177370.317756:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.317757:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177370.317759:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177370.317761:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177370.317765:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:2.0:1786177370.317769:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177370.317775:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177370.317783:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177370.317786:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177370.317790:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000088313762] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.321031:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177370.321040:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1786177370.321045:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177370.321049:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177370.321052:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177370.321068:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.321079:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177370.321081:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177370.321086:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177370.321090:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177370.321092:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177370.321095:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609088:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177370.321098:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177370.321104:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177370.321106:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177370.321109:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177370.321122:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177370.321124:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177370.321134:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177370.321141:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e528. 00000400:00000010:0.0:1786177370.321236:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686330. 00000100:00000200:0.0:1786177370.321246:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235609088, portal 10 00000100:00000001:0.0:1786177370.321249:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177370.321251:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469617408 : -110371239934208 : ffff9b9e31734700) 00000100:00000040:0.0:1786177370.321262:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.321271:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177370.321274:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177370.321277:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686550. 00000100:00000200:0.0:1786177370.321281:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235609088, offset 0 00000400:00000200:0.0:1786177370.321290:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177370.321303:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177370.321314:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177370.321320:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177370.321325:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177370.321329:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177370.321332:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177370.321337:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426825:426825:255:4294967295] 192.168.202.132@tcp LPNI seq info [426787:426787:7:4294967295] 00000400:00000200:0.0:1786177370.321342:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177370.321354:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177370.321382:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177370.321389:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c600. 00000800:00000200:0.0:1786177370.321398:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177370.321404:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177370.321409:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177370.321413:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177370.321416:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177370.321419:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306717191 00000100:00000001:0.0:1786177370.321422:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306721024 00000100:00000001:0.0:1786177370.321424:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306717191 00000100:00000001:0.0:1786177370.321426:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306721024 00000100:00000001:0.0:1786177370.321427:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177370.321429:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177370.321432:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177370.321435:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177370.321437:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177370.321438:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1786177370.321439:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177370.321440:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177370.321441:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177370.321442:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.326407:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177370.326423:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177370.326428:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177370.326431:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177370.326435:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177370.326440:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177370.326450:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12044 00000400:00000200:2.0:1786177370.326461:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177370.326469:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177370.326472:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177370.326476:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.326480:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177370.326483:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.326486:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177370.326491:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177370.326494:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177370.326497:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177370.326501:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177370.326504:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177370.326507:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177370.326510:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.326513:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177370.326515:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.326517:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177370.326520:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177370.326524:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177370.326526:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177370.326529:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177370.326538:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177370.326554:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.326557:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177370.326562:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177370.326565:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.326567:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177370.326587:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737480. 00000020:00000040:2.0:1786177370.326592:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177370.326597:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177370.326599:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.326603:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177370.326606:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177370.326610:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177370.326621:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaac00. 00000100:00000001:2.0:1786177370.326630:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177370.326641:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177370.326653:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177370.326658:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177370.327168:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046bba14a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177370.327184:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177370.327187:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177370.327191:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177370.327193:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177370.327195:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177370.327199:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177370.327201:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12044 00000400:00000200:2.0:1786177370.327207:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177370.327211:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177370.327213:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177370.327216:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.327219:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177370.327221:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.327224:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177370.327226:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177370.327227:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177370.327229:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177370.327232:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177370.327234:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177370.327236:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177370.327239:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.327241:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177370.327243:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.327245:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177370.327247:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177370.327250:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177370.327252:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177370.327255:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177370.327259:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177370.327261:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.327264:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177370.327267:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177370.327269:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.327271:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177370.327276:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735880. 00000020:00000040:2.0:1786177370.327278:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177370.327281:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177370.327282:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.327284:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177370.327286:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177370.327289:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177370.327294:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9800. 00000100:00000001:2.0:1786177370.327298:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177370.327303:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177370.327312:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177370.327314:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177370.327318:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000120b47b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177370.327326:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177370.327329:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177370.327332:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177370.327334:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177370.327335:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177370.327338:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177370.327340:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12044 00000400:00000200:2.0:1786177370.327344:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177370.327346:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177370.327348:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177370.327352:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.327354:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177370.327357:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.327394:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177370.327397:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177370.327398:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177370.327399:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177370.327402:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177370.327403:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177370.327405:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177370.327408:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.327409:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177370.327411:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177370.327413:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177370.327415:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177370.327418:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177370.327420:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177370.327422:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177370.327424:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177370.327426:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.327428:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177370.327429:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177370.327431:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.327433:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177370.327437:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735c00. 00000020:00000040:2.0:1786177370.327439:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177370.327442:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177370.327443:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.327445:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177370.327446:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177370.327448:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177370.327451:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177370.327455:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177370.327459:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177370.327467:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177370.327469:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177370.327473:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005c78d727] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177370.327480:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177370.327487:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12054) 00000100:00000001:0.0:1786177370.328503:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177370.328510:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177370.328513:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177370.328525:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.328537:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177370.329486:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177370.329498:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177370.329502:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177370.329506:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177370.329513:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609216:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177370.329519:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177370.329526:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177370.329529:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177370.329531:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177370.329542:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177370.329545:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177370.329553:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177370.329559:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610ebb0. 00000400:00000010:0.0:1786177370.329569:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686aa0. 00000100:00000200:0.0:1786177370.329650:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235609216, portal 10 00000100:00000001:0.0:1786177370.329655:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177370.329657:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469629056 : -110371239922560 : ffff9b9e31737480) 00000100:00000040:0.0:1786177370.329681:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.329689:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177370.329692:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177370.329696:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e316862a8. 00000100:00000200:0.0:1786177370.329699:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235609216, offset 0 00000400:00000200:0.0:1786177370.329708:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177370.329931:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177370.329943:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177370.330011:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177370.330018:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177370.330024:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177370.330027:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177370.330032:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426826:426826:254:4294967295] 192.168.202.132@tcp LPNI seq info [426788:426788:6:4294967295] 00000400:00000200:0.0:1786177370.330037:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177370.330050:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177370.330057:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177370.330064:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c300. 00000800:00000200:0.0:1786177370.330072:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177370.330079:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177370.330085:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177370.330090:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177370.330094:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177370.330097:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306717200 00000100:00000001:0.0:1786177370.330101:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306721024 00000100:00000001:0.0:1786177370.330104:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306717200 00000100:00000001:0.0:1786177370.330106:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306721024 00000100:00000001:0.0:1786177370.330108:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177370.330111:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177370.330115:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177370.330121:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.330129:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177370.330130:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177370.330133:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177370.330136:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177370.330138:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177370.330141:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609344:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177370.330145:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177370.330539:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177370.330542:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177370.330545:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177370.330551:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177370.330553:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177370.330558:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177370.330562:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e840. 00000400:00000010:0.0:1786177370.330567:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686440. 00000100:00000200:0.0:1786177370.330572:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235609344, portal 4 00000100:00000001:0.0:1786177370.332532:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177370.332535:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469621888 : -110371239929728 : ffff9b9e31735880) 00000100:00000040:0.0:1786177370.332557:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.332568:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177370.332571:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177370.332580:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686bb0. 00000100:00000200:0.0:1786177370.332587:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235609344, offset 0 00000400:00000200:0.0:1786177370.332594:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177370.332606:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177370.332615:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177370.332621:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177370.332626:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177370.332630:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177370.332633:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177370.332637:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426827:426827:253:4294967295] 192.168.202.132@tcp LPNI seq info [426789:426789:5:4294967295] 00000400:00000200:0.0:1786177370.332642:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177370.332652:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177370.332658:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177370.332675:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c900. 00000800:00000200:0.0:1786177370.332683:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177370.332689:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177370.332695:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177370.332700:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177370.332704:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177370.332708:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306717203 00000100:00000001:0.0:1786177370.332711:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306721024 00000100:00000001:0.0:1786177370.332714:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306717203 00000100:00000001:0.0:1786177370.332716:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306721024 00000100:00000001:0.0:1786177370.332718:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177370.332721:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177370.332725:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177370.332731:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.332738:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177370.332740:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177370.332744:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177370.332749:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177370.332752:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177370.332755:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609472:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177370.332759:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177370.332764:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177370.332767:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177370.332770:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177370.332776:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177370.332779:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177370.332784:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177370.332798:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e268. 00000400:00000010:0.0:1786177370.332804:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e316863b8. 00000100:00000200:0.0:1786177370.332809:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235609472, portal 4 00000100:00000001:0.0:1786177370.332811:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177370.332812:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469622784 : -110371239928832 : ffff9b9e31735c00) 00000100:00000040:0.0:1786177370.332820:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177370.332827:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177370.332829:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177370.332832:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e316864c8. 00000100:00000200:0.0:1786177370.332835:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235609472, offset 0 00000400:00000200:0.0:1786177370.332840:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177370.332844:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177370.332847:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177370.332861:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177370.332864:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177370.332866:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177370.332869:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177370.332872:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426828:426828:252:4294967295] 192.168.202.132@tcp LPNI seq info [426790:426790:4:4294967295] 00000400:00000200:0.0:1786177370.332877:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177370.332883:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177370.332887:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177370.332891:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177370.332894:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177370.332897:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177370.332900:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177370.332902:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177370.332905:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177370.332907:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306717205 00000100:00000001:0.0:1786177370.332910:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306721024 00000100:00000001:0.0:1786177370.332912:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306717205 00000100:00000001:0.0:1786177370.332914:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306721024 00000100:00000001:0.0:1786177370.332916:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177370.332918:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177370.332923:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177370.332928:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177370.332938:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177370.332941:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1786177370.332945:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177370.332947:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177370.332949:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177370.332952:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177370.828873:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177370.828884:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177370.828888:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177370.828895:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177370.828900:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177370.828905:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177370.828908:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177370.828912:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177370.828917:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177371.212533:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177371.212545:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177371.212553:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177371.212557:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177371.212560:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177371.212565:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177371.212569:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177371.212572:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177371.212575:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177371.212578:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177371.212582:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177371.212584:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177371.212602:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177371.212608:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177371.212612:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177371.212617:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177371.212622:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177371.212628:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177371.212631:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177371.212636:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177371.212640:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177371.212872:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177371.212876:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177371.212879:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177371.852528:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177371.852540:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177371.852544:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177371.852553:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177371.852558:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177371.852563:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177371.852566:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177371.852569:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177371.852573:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177372.236533:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177372.236545:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177372.236548:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177372.236555:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177372.236560:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177372.236566:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177372.236571:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177372.236575:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177372.236579:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177372.236661:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177372.236668:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177372.236691:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177372.236693:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177372.236696:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177372.236697:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177372.241767:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177372.241775:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177372.241780:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177372.876557:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177372.876571:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177372.876574:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177372.876582:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177372.876586:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177372.876593:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177372.876595:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177372.876599:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177372.876603:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177373.260506:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177373.260518:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177373.260526:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177373.260527:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177373.260532:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177373.260534:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177373.260671:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177373.260676:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177373.260679:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177373.260685:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177373.260689:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177373.260694:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.260706:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177373.260710:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.260713:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177373.260781:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177373.260787:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177373.260791:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1786177373.388991:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:2.0:1786177373.389053:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1786177373.389067:0:2382:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177373.389071:0:2382:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:2.0:1786177373.389077:0:2382:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177373.389083:0:2382:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1786177373.389089:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:2.0:1786177373.389104:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ca00. 00000400:00000200:2.0:1786177373.389119:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1786177373.389124:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1786177373.389140:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686088 00000400:00000010:2.0:1786177373.389145:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686088. 00000100:00000001:2.0:1786177373.389152:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177373.389164:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177373.389200:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177373.389206:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177373.389215:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177373.389217:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1786177373.389229:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:2.0:1786177373.389232:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c600. 00000400:00000200:2.0:1786177373.389237:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:1.0:1786177373.389239:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1786177373.389241:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1786177373.389246:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1786177373.389247:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686550 00000400:00000010:2.0:1786177373.389251:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686550. 00000100:00000001:1.0:1786177373.389254:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1786177373.389255:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177373.389262:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177373.389267:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177370/real 1786177373] req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177373.389279:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177373.389286:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177373.389288:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177373.389291:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1786177373.389296:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1786177373.389298:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e580. 00000100:00000001:2.0:1786177373.389299:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1786177373.389304:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686110 00000400:00000010:1.0:1786177373.389308:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686110. 00000800:00000100:2.0:1786177373.389309:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:2.0:1786177373.389314:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c300. 00000100:00000001:1.0:1786177373.389315:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1786177373.389320:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000200:1.0:1786177373.389322:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177373.389333:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177373.389339:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1786177373.389349:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177373.389351:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389354:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:2.0:1786177373.389355:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e316862a8 00000100:00000001:1.0:1786177373.389397:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1786177373.389398:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e316862a8. 00000100:00000001:2.0:1786177373.389403:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:1.0:1786177373.389406:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177373.389411:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389427:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177373.389430:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177373.389437:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1786177373.389437:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389465:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177373.389465:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177373.389469:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177373.389471:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389475:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177373.389478:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000100:2.0:1786177373.389480:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000100:00000001:1.0:1786177373.389483:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000800:00000010:2.0:1786177373.389485:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c900. 00000100:00000001:1.0:1786177373.389487:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:2.0:1786177373.389490:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00080000:1.0:1786177373.389492:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:2.0:1786177373.389498:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000100:00080000:1.0:1786177373.389501:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000400:00000200:2.0:1786177373.389503:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686bb0 00000100:00000001:1.0:1786177373.389505:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000010:2.0:1786177373.389507:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686bb0. 00000100:00000001:2.0:1786177373.389511:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177373.389517:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177373.389526:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177373.389556:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1786177373.389558:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235608960:192.168.202.132@tcp:250:kworker.0 00000100:00000040:2.0:1786177373.389564:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389566:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177373.389571:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1786177373.389577:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177373.389578:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736d80 x1872930235608960/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177373.389582:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177373.389589:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000800:00000100:2.0:1786177373.389592:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 02000000:00000001:1.0:1786177373.389594:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000010:2.0:1786177373.389597:0:2382:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 02000000:00000010:1.0:1786177373.389601:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 00000400:00000200:2.0:1786177373.389602:0:2382:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 02000000:00000001:1.0:1786177373.389608:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1786177373.389609:0:2382:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000020:00000001:1.0:1786177373.389614:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000400:00000200:2.0:1786177373.389614:0:2382:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e316864c8 00000020:00000040:1.0:1786177373.389619:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000400:00000010:2.0:1786177373.389619:0:2382:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e316864c8. 00000020:00000001:1.0:1786177373.389626:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 00000100:00000001:2.0:1786177373.389626:0:2382:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177373.389643:0:2382:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177373.389652:0:2382:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:1.0:1786177373.389655:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa000. 00000100:00000040:2.0:1786177373.389655:0:2382:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177395 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1786177373.389662:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1786177373.389666:0:2382:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177373.389667:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1786177373.389671:0:2382:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:1.0:1786177373.389672:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736d80. 00000100:00000001:1.0:1786177373.389679:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177373.389681:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389684:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177373.389686:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177373.389688:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177373.389689:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177373.389694:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177370/real 1786177373] req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177373.389708:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177373.389709:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177373.389712:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610ebb0. 00000400:00000200:1.0:1786177373.389714:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686aa0 00000400:00000010:1.0:1786177373.389716:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686aa0. 00000100:00000001:1.0:1786177373.389718:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177373.389721:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177373.389729:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389736:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177373.389737:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389739:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177373.389740:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177373.389744:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389751:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177373.389754:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389761:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177373.389763:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389765:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177373.389766:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389768:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177373.389770:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177373.389773:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177373.389777:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177373.389779:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177373.389783:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177373.389791:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609216:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177373.389795:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177373.389798:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177370/real 1786177373] req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177373.389815:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177373.389817:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177373.389819:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e840. 00000400:00000200:1.0:1786177373.389822:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686440 00000400:00000010:1.0:1786177373.389823:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686440. 00000100:00000001:1.0:1786177373.389825:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177373.389829:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177373.389836:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389841:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177373.389843:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389845:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177373.389846:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177373.389850:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389856:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177373.389859:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389866:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177373.389868:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389870:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177373.389872:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389873:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177373.389875:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177373.389878:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177373.389881:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177373.389883:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177373.389887:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177373.389895:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609344:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177373.389899:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177373.389903:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177370/real 1786177373] req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177373.389909:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177373.389910:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177373.389912:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e268. 00000400:00000200:1.0:1786177373.389915:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e316863b8 00000400:00000010:1.0:1786177373.389916:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e316863b8. 00000100:00000001:1.0:1786177373.389918:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177373.389921:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177373.389928:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389934:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177373.389935:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389937:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177373.389938:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177373.389942:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389952:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177373.389955:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.389962:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177373.389964:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389966:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177373.389967:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.389969:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177373.389971:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177373.389973:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177373.389976:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177373.389978:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177373.389982:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177373.389990:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609472:192.168.202.132@tcp:8:kworker.0 00000100:00000001:1.0:1786177373.389994:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1786177373.389998:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177370/real 1786177373] req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1786177373.390005:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1786177373.390006:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1786177373.390008:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e528. 00000400:00000200:1.0:1786177373.390010:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686330 00000400:00000010:1.0:1786177373.390012:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686330. 00000100:00000001:1.0:1786177373.390014:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1786177373.390017:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1786177373.390029:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.390035:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177373.390036:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.390038:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177373.390039:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1786177373.390042:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.390047:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177373.390051:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.390057:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177373.390059:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.390060:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1786177373.390062:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.390063:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1786177373.390066:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1786177373.390068:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1786177373.390071:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177373.390074:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1786177373.390077:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1786177373.390085:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609088:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177373.390089:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.390091:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177373.390094:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737480 x1872930235609216/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.390104:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177373.390106:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177373.390108:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:1.0:1786177373.390111:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177373.390112:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177373.390113:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177373.390115:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177373.390117:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:1.0:1786177373.390119:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177373.390121:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177373.390122:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737480. 00000100:00000001:1.0:1786177373.390125:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177373.390126:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.390128:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177373.390131:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735880 x1872930235609344/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.390138:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177373.390139:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177373.390141:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:1.0:1786177373.390144:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177373.390145:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177373.390146:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177373.390148:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177373.390149:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9800. 02000000:00000001:1.0:1786177373.390151:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177373.390152:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177373.390153:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735880. 00000100:00000001:1.0:1786177373.390155:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177373.390157:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.390158:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177373.390162:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735c00 x1872930235609472/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.390168:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177373.390170:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177373.390171:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:1.0:1786177373.390173:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177373.390174:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177373.390176:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177373.390177:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177373.390179:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:1.0:1786177373.390181:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177373.390182:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177373.390183:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735c00. 00000100:00000001:1.0:1786177373.390185:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177373.390186:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.390188:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177373.390191:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734700 x1872930235609088/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177395 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177373.390197:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177373.390198:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177373.390199:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:1.0:1786177373.390201:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177373.390202:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177373.390203:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:1.0:1786177373.390205:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177373.390206:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:1.0:1786177373.390208:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177373.390209:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177373.390211:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734700. 00000100:00000001:1.0:1786177373.390213:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177373.390214:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.390216:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177373.390225:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177373.390227:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177373.390230:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177373.390231:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177373.900858:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177373.900867:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177373.900870:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177373.900877:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177373.900881:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177373.900885:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177373.900888:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177373.900891:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177373.900895:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177374.220677:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306721024, at 4306721088 00000100:00000010:1.0:1786177374.220699:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d56c0. 00000100:00000040:0.0:1786177374.221350:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306721024, at 4306721088 00000100:00000010:0.0:1786177374.221387:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5060. 00000100:00000001:1.0:1786177374.284521:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177374.284531:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177374.284535:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177374.284541:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177374.284546:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177374.284551:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177374.284554:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177374.284557:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177374.284560:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177374.287038:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177374.287049:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177374.287052:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177374.287057:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177374.287064:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177374.287068:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177374.287070:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177374.287073:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177374.287078:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177374.287080:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177374.287084:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177374.287098:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177374.287100:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177374.287102:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177374.287104:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177374.287105:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177374.287107:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177374.287109:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177374.287439:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177374.287443:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177374.287446:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177374.924521:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177374.924533:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177374.924536:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177374.924544:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177374.924550:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177374.924555:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177374.924558:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177374.924561:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177374.924566:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.308473:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177375.308481:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177375.308484:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177375.308490:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177375.308494:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177375.308499:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177375.308502:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177375.308505:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177375.308509:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177375.308514:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177375.308521:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177375.308528:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177375.308529:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177375.308533:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177375.308535:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177375.309077:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177375.309082:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177375.309086:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1786177375.437040:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177375.437056:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177375.437059:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177375.437065:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177375.437071:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177375.437081:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12049 00000400:00000200:1.0:1786177375.437094:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177375.437451:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177375.437456:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177375.437461:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.437465:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177375.437468:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.437471:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177375.437476:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177375.437478:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177375.437482:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177375.437486:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177375.437489:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177375.437493:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177375.437496:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.437498:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177375.437500:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.437503:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177375.437506:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177375.437509:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177375.437512:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177375.437516:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177375.437521:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177375.437524:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.437527:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177375.437530:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177375.437542:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8e00. 00000020:00000040:1.0:1786177375.437547:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177375.437552:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177375.437554:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.437558:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177375.437562:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177375.437567:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177375.437577:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac000. 00000100:00000001:1.0:1786177375.437586:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177375.437599:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177375.438542:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177375.438553:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177375.438564:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ba93daf1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177375.438601:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177375.438605:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177375.438621:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177375.438625:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177375.438627:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177375.438631:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177375.438634:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12049 00000400:00000200:1.0:1786177375.438642:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177375.438647:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177375.438650:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177375.438653:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.438655:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177375.438657:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.438659:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177375.438662:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177375.438664:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177375.438667:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177375.438670:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177375.438672:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177375.438674:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177375.438678:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.438679:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177375.438681:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.438682:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177375.438685:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:1.0:1786177375.438688:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177375.438691:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177375.438693:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177375.438701:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177375.438703:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.438705:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177375.438710:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177375.438713:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.438716:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177375.438723:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8700. 00000020:00000040:1.0:1786177375.438727:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177375.438731:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177375.438733:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.438735:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177375.438739:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177375.438742:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177375.438747:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac400. 00000100:00000001:1.0:1786177375.438753:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177375.438760:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177375.438769:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177375.438772:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177375.438776:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007a756761] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177375.438784:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.438785:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1786177375.438788:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1786177375.438797:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1786177375.438797:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177375.438799:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1786177375.438804:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1786177375.438805:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177375.438814:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177375.438818:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12049 00000400:00000200:1.0:1786177375.438824:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:0.0:1786177375.438831:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1786177375.438833:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177375.438840:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177375.438850:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177375.438863:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177375.438863:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1786177375.438872:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177375.438873:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.438880:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 02000000:00000001:0.0:1786177375.438882:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1786177375.438884:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177375.438888:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 02000000:00000001:0.0:1786177375.438889:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1786177375.438891:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177375.438895:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 02000000:00000001:0.0:1786177375.438897:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177375.438900:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177375.438910:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00100000:0.0:1786177375.438911:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235609600:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177375.438922:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177375.438927:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177375.438930:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177375.438933:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177375.438941:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177375.438943:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177375.438953:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177375.438958:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177375.438966:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177375.438973:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235609600, portal 25 00000100:00000001:0.0:1786177375.438975:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177375.438978:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487985664 : -110371221565952 : ffff9b9e328b8e00) 00000100:00000040:0.0:1786177375.438985:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.438993:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177375.438996:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177375.438999:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177375.439001:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235609600, offset 0 00000400:00000200:0.0:1786177375.439008:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177375.439017:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177375.439025:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177375.439030:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177375.439035:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177375.439038:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177375.439041:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177375.439044:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426829:426829:256:4294967295] 192.168.202.132@tcp LPNI seq info [426791:426791:8:4294967295] 00000400:00000200:0.0:1786177375.439048:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177375.439058:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177375.439064:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177375.439070:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177375.439077:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177375.439081:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1786177375.439086:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177375.439090:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000100:0.0:1786177375.439095:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177375.439098:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177375.439116:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177375.439121:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177375.439127:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177375.439129:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177375.439134:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177375.439138:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439149:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177375.439153:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439160:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.439162:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177375.439168:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177375.439172:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177375.439175:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306726146 00000100:00000010:0.0:1786177375.439181:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5060. 00000100:00000001:0.0:1786177375.439187:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177375.439195:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306726146 00000100:00000010:0.0:1786177375.439198:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5000. 00000100:00000001:0.0:1786177375.439202:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177375.439205:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177375.439207:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.439209:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177375.439214:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177375/real 1786177375] req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177375.439223:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177375.439225:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177375.439226:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177375.439228:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177375.439230:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177375.439232:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177375.439235:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177375.439243:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439249:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177375.439250:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.439253:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177375.439255:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177375.439258:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439266:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177375.439269:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439276:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177375.439278:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.439280:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177375.439281:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.439283:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177375.439285:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177375.439289:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177375.439292:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177375.439295:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177375.439299:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177375.439307:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235609600:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177375.439311:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177375.439315:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439322:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177375.439323:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177375.439326:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177375.439328:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177375.439330:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177375.439333:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609728:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177375.439337:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177375.439339:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177375.439340:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177375.439343:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177375.439346:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177375.439434:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177375.439478:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177375.439481:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177375.439485:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177375.439488:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235609728, portal 10 00000100:00000001:0.0:1786177375.439490:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177375.439492:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487983872 : -110371221567744 : ffff9b9e328b8700) 00000100:00000040:0.0:1786177375.439499:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439505:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177375.439507:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177375.439510:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177375.439512:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235609728, offset 0 00000400:00000200:0.0:1786177375.439516:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177375.439520:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177375.439524:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177375.439528:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177375.439531:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177375.439533:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177375.439535:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177375.439538:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426830:426830:256:4294967295] 192.168.202.132@tcp LPNI seq info [426792:426792:8:4294967295] 00000400:00000200:0.0:1786177375.439542:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177375.439548:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177375.439552:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177375.439555:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177375.439559:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177375.439561:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1786177375.439565:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177375.439568:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000100:0.0:1786177375.439572:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177375.439602:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177375.439606:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177375.439646:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177375.439650:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177375.439652:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177375.439655:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177375.439673:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439703:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177375.439708:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439714:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.439715:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177375.439719:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177375.439724:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177375.439744:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177375.439746:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306722310 00000100:00000001:0.0:1786177375.439749:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306726146 00000100:00000001:0.0:1786177375.439751:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306722310 00000100:00000001:0.0:1786177375.439753:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306726146 00000100:00000001:0.0:1786177375.439755:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177375.439757:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.439760:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177375.439764:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177375/real 1786177375] req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177375.439773:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177375.439774:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177375.439776:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177375.439778:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177375.439780:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177375.439782:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177375.439786:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177375.439795:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439801:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177375.439803:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.439805:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177375.439806:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177375.439809:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439815:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177375.439819:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439826:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177375.439828:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.439830:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177375.439831:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.439833:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177375.439835:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177375.439838:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177375.439841:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177375.439843:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177375.439847:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177375.439855:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609728:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177375.439859:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.439864:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177375.439867:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8e00 x1872930235609600/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439873:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177375.439876:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177375.439878:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177375.439883:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177375.439886:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177375.439888:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177375.439890:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177375.439892:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac000. 02000000:00000001:0.0:1786177375.439896:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177375.439898:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177375.439900:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8e00. 00000100:00000001:0.0:1786177375.439903:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177375.439904:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.439907:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177375.439911:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8700 x1872930235609728/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.439918:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177375.439920:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177375.439922:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177375.439924:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177375.439925:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177375.439926:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177375.439928:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177375.439930:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac400. 02000000:00000001:0.0:1786177375.439932:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177375.439934:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177375.439936:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8700. 00000100:00000001:0.0:1786177375.439938:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177375.439940:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.439943:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177375.439946:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177375.439953:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177375.439954:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1786177375.439958:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177375.439960:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177375.439965:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1786177375.439967:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177375.439970:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1786177375.439973:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:1.0:1786177375.439981:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177375.439984:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177375.439986:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177375.439993:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177375.439996:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.439998:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177375.440001:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177375.440004:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.440006:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177375.440013:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8700. 00000020:00000040:1.0:1786177375.440016:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177375.440021:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177375.440022:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.440025:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177375.440028:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177375.440031:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177375.440039:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae800. 00000100:00000001:1.0:1786177375.440044:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177375.440051:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177375.440060:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177375.440064:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177375.440068:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007a756761] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177375.440091:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177375.440094:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177375.440108:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177375.440109:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177375.440111:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177375.440115:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177375.440118:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12049 00000400:00000200:1.0:1786177375.440124:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177375.440128:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177375.440131:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177375.440133:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.440136:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177375.440138:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.440140:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177375.440148:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177375.440149:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177375.440151:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177375.440154:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177375.440156:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177375.440159:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:1.0:1786177375.440162:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.440165:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177375.440168:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.440171:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177375.440175:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:1.0:1786177375.440178:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177375.440180:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177375.440183:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177375.440186:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177375.440189:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.440191:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177375.440194:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177375.440197:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.440199:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177375.440204:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8e00. 00000020:00000040:1.0:1786177375.440207:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177375.440211:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177375.440212:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.440214:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177375.440219:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177375.440221:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177375.440227:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.440235:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177375.440236:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177375.440239:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177375.440241:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177375.440243:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177375.440246:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609856:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177375.440249:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177375.440253:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177375.440254:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177375.440256:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177375.440260:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177375.440262:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177375.440266:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177375.440269:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177375.440272:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177375.440276:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235609856, portal 10 00000100:00000001:0.0:1786177375.440278:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177375.440279:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487983872 : -110371221567744 : ffff9b9e328b8700) 00000100:00000040:0.0:1786177375.440285:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.440290:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177375.440292:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177375.440294:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177375.440296:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235609856, offset 0 00000400:00000200:0.0:1786177375.440300:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177375.440304:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177375.440308:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177375.440311:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177375.440314:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177375.440318:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177375.440320:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177375.440322:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426831:426831:256:4294967295] 192.168.202.132@tcp LPNI seq info [426793:426793:8:4294967295] 00000400:00000200:0.0:1786177375.440326:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177375.440331:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177375.440335:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177375.440339:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177375.440343:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177375.440345:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1786177375.440349:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177375.440352:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000100:0.0:1786177375.440354:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177375.440372:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177375.440376:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177375.440379:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177375.440382:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177375.440384:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177375.440386:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177375.440389:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.440430:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177375.440434:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.440440:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.440442:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177375.440446:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177375.440449:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177375.440453:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177375.440454:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306722310 00000100:00000001:0.0:1786177375.440457:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306726146 00000100:00000001:0.0:1786177375.440459:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306722310 00000100:00000001:0.0:1786177375.440462:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306726146 00000100:00000001:0.0:1786177375.440464:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177375.440466:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.440469:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177375.440474:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177375/real 1786177375] req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177375.440481:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177375.440483:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1786177375.440496:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1786177375.440497:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 02000000:00000001:1.0:1786177375.440501:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1786177375.440504:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 02000000:00000001:1.0:1786177375.440506:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1786177375.440509:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177375.440515:0:2392:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000010:1.0:1786177375.440517:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae400. 00000100:00000200:0.0:1786177375.440519:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177375.440524:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177375.440539:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1786177375.440540:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177375.440572:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177375.440575:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1786177375.440578:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.440580:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.440584:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177375.440589:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1786177375.440589:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ba93daf1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177375.440605:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177375.440625:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177375.440635:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1786177375.440636:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1786177375.440646:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000040:0.0:1786177375.440650:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177375.440650:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177375.440655:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177375.440662:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177375.440671:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12049 00000100:00000001:0.0:1786177375.440676:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:1.0:1786177375.440682:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177375.440684:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.440689:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:1.0:1786177375.440691:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00000001:0.0:1786177375.440693:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1786177375.440696:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1786177375.440698:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:1.0:1786177375.440703:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177375.440704:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1786177375.440710:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:0.0:1786177375.440713:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00000040:1.0:1786177375.440717:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00080000:0.0:1786177375.440723:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:1.0:1786177375.440723:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177375.440731:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:1.0:1786177375.440731:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177375.440734:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177375.440738:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177375.440742:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000100:00000040:0.0:1786177375.440743:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1786177375.440748:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:1.0:1786177375.440755:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:1.0:1786177375.440764:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00100000:0.0:1786177375.440772:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609856:192.168.202.132@tcp:38:kworker.0 00000100:00000001:1.0:1786177375.440772:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177375.440777:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177375.440783:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:0.0:1786177375.440784:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1786177375.440792:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00000001:0.0:1786177375.440793:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1786177375.440801:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000040:0.0:1786177375.440802:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8700 x1872930235609856/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177375.440809:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177375.440815:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177375.440821:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177375.440826:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177375.440829:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177375.440830:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1786177375.440833:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000010:0.0:1786177375.440836:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 00000008:00100000:1.0:1786177375.440838:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 02000000:00000001:0.0:1786177375.440842:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00000001:1.0:1786177375.440844:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1786177375.440846:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000100:00000001:1.0:1786177375.440849:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1786177375.440851:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000010:1.0:1786177375.440857:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9c00. 00000020:00000001:0.0:1786177375.440858:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177375.440862:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae800. 00000020:00000040:1.0:1786177375.440865:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177375.440868:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177375.440871:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177375.440872:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:1.0:1786177375.440875:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177375.440878:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8700. 00000100:00000001:1.0:1786177375.440880:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177375.440884:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1786177375.440885:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000001:1.0:1786177375.440888:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.440889:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.440895:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1786177375.440897:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ae800. 00000100:00000001:0.0:1786177375.440900:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177375.440904:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177375.440904:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.440913:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177375.440917:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1786177375.440918:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177375.440921:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177375.440931:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177375.440933:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177375.440938:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177375.440948:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3ea07ed] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.440954:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177375.440960:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177375.440966:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1786177375.440970:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177375.440971:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177375.440980:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177375.440984:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12060) 00000100:00100000:0.0:1786177375.440985:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609984:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177375.440992:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177375.440996:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177375.440998:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177375.441000:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177375.441005:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177375.441008:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177375.441013:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177375.441016:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177375.441022:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177375.441026:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235609984, portal 4 00000100:00000001:0.0:1786177375.441027:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177375.441029:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487985664 : -110371221565952 : ffff9b9e328b8e00) 00000100:00000040:0.0:1786177375.441037:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441043:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177375.441045:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177375.441047:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177375.441050:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235609984, offset 0 00000400:00000200:0.0:1786177375.441054:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177375.441058:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177375.441063:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177375.441066:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177375.441070:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177375.441072:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177375.441075:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177375.441078:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426832:426832:256:4294967295] 192.168.202.132@tcp LPNI seq info [426794:426794:8:4294967295] 00000400:00000200:0.0:1786177375.441082:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177375.441088:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177375.441091:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177375.441094:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177375.441098:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177375.441101:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1786177375.441106:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177375.441223:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000100:0.0:1786177375.441227:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177375.441229:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177375.441232:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177375.441236:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177375.441240:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177375.441241:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177375.441243:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177375.441247:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441254:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177375.441256:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441261:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.441262:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177375.441266:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177375.441269:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177375.441272:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177375.441274:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306722311 00000100:00000001:0.0:1786177375.441277:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306726146 00000100:00000001:0.0:1786177375.441279:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306722311 00000100:00000001:0.0:1786177375.441280:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306726146 00000100:00000001:0.0:1786177375.441282:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177375.441284:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.441286:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177375.441289:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177375/real 1786177375] req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177375.441296:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177375.441297:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177375.441299:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177375.441301:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177375.441303:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177375.441305:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177375.441307:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177375.441315:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441321:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177375.441322:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441324:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177375.441325:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177375.441328:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441334:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177375.441338:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441344:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177375.441346:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441347:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177375.441349:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441351:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177375.441352:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177375.441355:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177375.441383:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177375.441386:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177375.441390:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177375.441397:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235609984:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177375.441401:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441403:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177375.441406:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8e00 x1872930235609984/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441412:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177375.441414:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177375.441416:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177375.441418:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177375.441420:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177375.441421:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177375.441423:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177375.441425:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae400. 02000000:00000001:0.0:1786177375.441427:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177375.441428:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177375.441429:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8e00. 00000100:00000001:0.0:1786177375.441432:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177375.441433:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441434:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441438:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177375.441439:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.441441:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177375.441442:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177375.441444:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177375.441448:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441455:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177375.441456:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177375.441458:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177375.441460:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177375.441462:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177375.441464:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235610112:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177375.441468:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177375.441470:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177375.441471:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177375.441473:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177375.441476:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177375.441477:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177375.441480:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177375.441483:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177375.441486:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177375.441489:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235610112, portal 4 00000100:00000001:0.0:1786177375.441491:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177375.441492:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487989248 : -110371221562368 : ffff9b9e328b9c00) 00000100:00000040:0.0:1786177375.441497:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441504:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177375.441506:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177375.441507:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177375.441510:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235610112, offset 0 00000400:00000200:0.0:1786177375.441513:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177375.441516:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177375.441520:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177375.441523:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177375.441525:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177375.441528:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177375.441529:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177375.441532:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426833:426833:256:4294967295] 192.168.202.132@tcp LPNI seq info [426795:426795:8:4294967295] 00000400:00000200:0.0:1786177375.441536:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177375.441542:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177375.441547:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177375.441550:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177375.441554:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177375.441556:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1786177375.441560:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177375.441564:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 57 secs later) 00000800:00000100:0.0:1786177375.441567:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177375.441569:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177375.441572:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177375.441575:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177375.441578:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177375.441579:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177375.441582:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177375.441585:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441592:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177375.441594:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177400 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441600:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.441602:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177375.441605:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177375.441608:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177375.441621:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177375.441623:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306722314 00000100:00000001:0.0:1786177375.441625:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306726146 00000100:00000001:0.0:1786177375.441627:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306722314 00000100:00000001:0.0:1786177375.441629:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306726146 00000100:00000001:0.0:1786177375.441630:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177375.441632:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.441634:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177375.441638:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177375/real 1786177375] req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177375.441645:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177375.441646:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177375.441648:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177375.441651:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177375.441652:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177375.441654:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177375.441657:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177375.441665:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441671:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177375.441673:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441674:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177375.441676:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177375.441678:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441685:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177375.441688:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441694:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177375.441696:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441698:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177375.441699:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441701:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177375.441702:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177375.441705:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177375.441708:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177375.441711:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177375.441715:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177375.441723:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235610112:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177375.441726:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441728:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177375.441731:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9c00 x1872930235610112/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177400 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177375.441738:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177375.441739:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177375.441741:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177375.441743:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177375.441745:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177375.441746:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177375.441748:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177375.441750:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ae800. 02000000:00000001:0.0:1786177375.441752:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177375.441753:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177375.441754:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9c00. 00000100:00000001:0.0:1786177375.441756:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177375.441758:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441759:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177375.441762:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177375.441764:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177375.441765:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177375.441767:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177375.948516:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177375.948527:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177375.948529:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177375.948535:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177375.948539:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177375.948544:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177375.948546:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177375.948548:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177375.948551:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177376.332416:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177376.332424:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177376.332426:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177376.332432:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177376.332436:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177376.332439:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177376.332442:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177376.332444:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177376.332447:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177376.333456:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177376.333461:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177376.333465:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177376.333466:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177376.333469:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177376.333471:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177376.333856:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177376.333861:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177376.333864:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177376.972469:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177376.972478:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177376.972481:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177376.972486:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177376.972491:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177376.972494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177376.972496:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177376.972498:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177376.972501:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177377.356461:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177377.356468:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177377.356474:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177377.356476:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177377.356480:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177377.356481:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177377.356482:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177377.356494:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177377.356497:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177377.356501:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177377.356518:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177377.356522:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177377.356525:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177377.356527:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177377.356530:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177377.356637:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177377.356641:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177377.356644:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177377.998693:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177377.998701:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177377.998703:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177377.998708:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177377.998712:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177377.998716:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177377.998718:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177377.998720:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177377.998724:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177378.380475:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177378.380482:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177378.380488:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177378.380489:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177378.380493:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177378.380494:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177378.380621:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177378.380624:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177378.380627:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177378.380794:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177378.380798:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177378.380800:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177378.380804:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177378.380807:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177378.380810:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177378.380813:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177378.380815:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177378.380817:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177379.020454:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177379.020461:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177379.020463:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177379.020468:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177379.020472:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177379.020476:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177379.020479:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177379.020481:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177379.020484:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177379.340467:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306726146, at 4306726208 00000100:00000010:1.0:1786177379.340482:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5000. 00000100:00000040:0.0:1786177379.340483:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306726146, at 4306726208 00000100:00000010:0.0:1786177379.340492:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5060. 00000100:00000001:1.0:1786177379.404462:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177379.404467:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177379.404469:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177379.404472:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177379.404473:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177379.404480:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177379.404481:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177379.404484:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177379.404486:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177379.404489:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177379.404491:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177379.404492:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177379.404495:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177379.404498:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177379.404501:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177379.404624:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177379.404627:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177379.404630:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177380.046801:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177380.046809:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177380.046811:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177380.046816:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177380.046819:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177380.046822:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.046825:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177380.046827:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.046830:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177380.428467:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177380.428472:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177380.428477:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177380.428478:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177380.428482:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177380.428483:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177380.428586:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177380.428590:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177380.428593:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177380.430046:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177380.430051:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177380.430053:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177380.430058:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177380.430061:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177380.430065:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177380.430067:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177380.430070:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177380.430073:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177380.561494:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177380.561505:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177380.561507:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177380.561511:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177380.561515:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177380.561523:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12055 00000400:00000200:1.0:1786177380.561532:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177380.561539:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177380.561542:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177380.561545:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561549:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177380.561552:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561554:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177380.561559:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177380.561571:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177380.561574:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177380.561577:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177380.561580:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177380.561582:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177380.561586:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561588:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177380.561590:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561591:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177380.561594:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177380.561597:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561600:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177380.561603:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177380.561607:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177380.561611:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.561614:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177380.561617:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177380.561623:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9f80. 00000020:00000040:1.0:1786177380.561627:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177380.561631:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177380.561633:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.561636:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177380.561639:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177380.561642:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177380.561649:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ac800. 00000100:00000001:1.0:1786177380.561655:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177380.561664:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177380.561675:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177380.561679:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177380.561685:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c4f3e5a2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177380.561705:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177380.561708:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177380.561711:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177380.561713:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177380.561714:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177380.561717:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177380.561720:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12055 00000400:00000200:1.0:1786177380.561723:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177380.561726:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177380.561728:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177380.561732:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561734:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177380.561736:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561738:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177380.561742:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177380.561743:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177380.561744:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177380.561747:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177380.561749:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177380.561751:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177380.561755:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561758:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177380.561760:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561762:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177380.561765:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:1.0:1786177380.561768:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561771:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177380.561773:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177380.561788:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177380.561791:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.561793:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177380.561796:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177380.561799:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.561801:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177380.561804:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb480. 00000020:00000040:1.0:1786177380.561806:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177380.561808:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177380.561810:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.561811:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177380.561813:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177380.561814:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177380.561818:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261acc00. 00000100:00000001:1.0:1786177380.561821:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177380.561826:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177380.561834:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177380.561836:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177380.561840:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b192a357] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177380.561848:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177380.561850:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177380.561853:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177380.561854:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177380.561856:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177380.561858:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177380.561860:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12055 00000400:00000200:1.0:1786177380.561864:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177380.561866:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177380.561868:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177380.561870:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561872:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177380.561874:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561876:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177380.561878:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177380.561880:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:1.0:1786177380.561881:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177380.561883:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177380.561884:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177380.561887:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177380.561889:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561891:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177380.561893:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561895:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177380.561897:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:1.0:1786177380.561900:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561902:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177380.561904:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177380.561906:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177380.561909:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.561910:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177380.561912:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177380.561915:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.561916:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177380.561919:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328ba680. 00000020:00000040:1.0:1786177380.561921:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177380.561923:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177380.561924:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.561926:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177380.561927:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177380.561929:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177380.561932:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261adc00. 00000100:00000001:1.0:1786177380.561935:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177380.561939:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177380.561946:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177380.561948:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177380.561951:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000497a117c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177380.561959:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177380.561961:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177380.561964:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1786177380.561966:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177380.561967:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177380.561970:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177380.561972:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12055 00000400:00000200:1.0:1786177380.561976:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177380.561979:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177380.561981:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177380.561983:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561985:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177380.561987:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.561989:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177380.561991:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177380.561993:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177380.561994:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177380.561996:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177380.561998:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177380.562000:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:1.0:1786177380.562002:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.562004:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177380.562005:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.562007:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177380.562009:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:1.0:1786177380.562012:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177380.562014:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177380.562015:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177380.562018:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177380.562021:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.562022:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177380.562024:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177380.562204:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.562208:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177380.562211:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb100. 00000020:00000040:1.0:1786177380.562214:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177380.562217:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177380.562218:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.562220:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177380.562222:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177380.562224:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177380.562228:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af400. 00000100:00000001:1.0:1786177380.562231:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177380.562236:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177380.562242:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177380.562245:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177380.562248:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f57195c8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177380.562255:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177380.562258:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177380.562261:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177380.562262:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177380.562264:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177380.562266:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177380.562269:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12055 00000400:00000200:1.0:1786177380.562272:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177380.562275:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177380.562277:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177380.562280:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.562282:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177380.562284:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.562285:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177380.562287:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177380.562288:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177380.562290:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177380.562292:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:1.0:1786177380.562294:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:1.0:1786177380.562296:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:1.0:1786177380.562299:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.562301:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177380.562339:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177380.562341:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177380.562344:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:1.0:1786177380.562346:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177380.562348:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177380.562350:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177380.562353:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177380.562355:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.562411:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177380.562414:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1786177380.562417:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.562418:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177380.562421:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bb800. 00000020:00000040:1.0:1786177380.562424:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:1.0:1786177380.562426:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177380.562427:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177380.562429:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177380.562430:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177380.562432:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177380.562435:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261afc00. 00000100:00000001:1.0:1786177380.562438:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177380.562443:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177380.562450:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177380.562452:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177380.562456:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000008148c2a] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177380.562463:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177380.562468:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12065) 00000100:00000001:2.0:1786177380.562524:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177380.562528:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177380.562531:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177380.562537:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.562546:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177380.562548:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177380.562551:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177380.562554:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177380.562557:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177380.562571:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235610240:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177380.562576:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177380.562579:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177380.562581:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177380.562584:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177380.562589:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177380.562591:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177380.562596:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177380.562600:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177380.562606:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177380.562613:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235610240, portal 25 00000100:00000001:2.0:1786177380.562615:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177380.562617:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487990144 : -110371221561472 : ffff9b9e328b9f80) 00000100:00000040:2.0:1786177380.562623:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.562631:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177380.562635:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177380.562638:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177380.562641:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235610240, offset 0 00000400:00000200:2.0:1786177380.562648:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177380.562655:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177380.562662:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177380.562667:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177380.562671:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177380.562675:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177380.562678:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177380.562682:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426834:426834:256:4294967295] 192.168.202.132@tcp LPNI seq info [426796:426796:8:4294967295] 00000400:00000200:2.0:1786177380.562687:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177380.562698:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177380.562705:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177380.562710:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177380.562717:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177380.562721:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786177380.562727:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177380.562730:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786177380.562734:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177380.562736:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177380.562740:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177380.562746:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177380.562751:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177380.562754:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177380.562758:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177380.562763:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.562773:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177380.562776:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.562794:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177380.562796:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177380.562802:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177380.562806:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177380.562808:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306731270 00000100:00000010:2.0:1786177380.562813:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c540. 00000100:00000001:2.0:1786177380.562819:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177380.562824:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306731270 00000100:00000010:2.0:1786177380.562827:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c120. 00000100:00000001:2.0:1786177380.562831:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177380.562834:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177380.562836:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177380.562838:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177380.562843:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177380/real 1786177380] req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177380.562852:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177380.562854:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177380.562856:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177380.562858:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177380.562860:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177380.562862:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177380.562865:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177380.562873:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.562880:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177380.562881:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.562883:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177380.562885:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177380.562889:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.562897:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177380.562902:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.562909:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177380.562912:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.562914:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177380.562915:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.562918:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177380.562920:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177380.562922:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177380.562926:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177380.562929:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177380.562933:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177380.562940:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235610240:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177380.562944:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177380.562948:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.562954:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177380.562955:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177380.562957:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177380.562959:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177380.562961:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177380.562964:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235610368:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177380.562967:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177380.562969:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177380.562970:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177380.562972:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177380.562975:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177380.562977:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177380.562981:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177380.562983:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177380.562986:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177380.562989:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235610368, portal 10 00000100:00000001:2.0:1786177380.562991:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177380.562992:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487995520 : -110371221556096 : ffff9b9e328bb480) 00000100:00000040:2.0:1786177380.562998:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563004:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177380.563007:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177380.563009:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177380.563011:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235610368, offset 0 00000400:00000200:2.0:1786177380.563014:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177380.563018:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177380.563021:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177380.563024:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177380.563026:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177380.563039:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177380.563042:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177380.563045:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426835:426835:256:4294967295] 192.168.202.132@tcp LPNI seq info [426797:426797:8:4294967295] 00000400:00000200:2.0:1786177380.563050:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177380.563056:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177380.563060:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177380.563063:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177380.563067:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177380.563070:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786177380.563073:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177380.563076:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786177380.563079:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177380.563081:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177380.563084:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177380.563087:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177380.563090:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177380.563091:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177380.563093:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177380.563097:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563110:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177380.563113:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563120:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177380.563121:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177380.563124:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177380.563128:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177380.563131:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177380.563133:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306727433 00000100:00000001:2.0:1786177380.563135:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306731270 00000100:00000001:2.0:1786177380.563137:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306727433 00000100:00000001:2.0:1786177380.563140:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306731270 00000100:00000001:2.0:1786177380.563142:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177380.563144:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177380.563145:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177380.563149:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177380/real 1786177380] req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177380.563156:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177380.563157:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177380.563159:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177380.563161:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177380.563163:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177380.563164:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177380.563167:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177380.563175:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563181:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177380.563183:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.563185:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177380.563186:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177380.563190:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563196:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177380.563200:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563207:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177380.563210:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.563211:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177380.563212:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.563214:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177380.563216:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177380.563218:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177380.563221:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177380.563223:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177380.563227:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177380.563234:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235610368:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177380.563238:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177380.563242:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563249:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177380.563250:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177380.563252:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177380.563254:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177380.563256:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177380.563260:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235610496:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177380.563263:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177380.563265:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177380.563267:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177380.563269:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177380.563272:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786177380.563275:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177380.563278:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177380.563280:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177380.563282:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177380.563284:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235610496, portal 10 00000100:00000001:2.0:1786177380.563286:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177380.563288:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487991936 : -110371221559680 : ffff9b9e328ba680) 00000100:00000040:2.0:1786177380.563293:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563298:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177380.563300:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177380.563302:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177380.563304:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235610496, offset 0 00000400:00000200:2.0:1786177380.563308:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177380.563312:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177380.563315:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177380.563318:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177380.563320:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177380.563323:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177380.563325:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177380.563328:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426836:426836:256:4294967295] 192.168.202.132@tcp LPNI seq info [426798:426798:8:4294967295] 00000400:00000200:2.0:1786177380.563333:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177380.563338:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177380.563342:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177380.563345:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177380.563349:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177380.563352:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786177380.563356:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177380.563376:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786177380.563401:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177380.563404:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177380.563407:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177380.563411:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177380.563414:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177380.563416:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177380.563418:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177380.563423:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563430:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177380.563433:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563440:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177380.563441:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177380.563444:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177380.563447:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177380.563449:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177380.563451:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306727434 00000100:00000001:2.0:1786177380.563454:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306731270 00000100:00000001:2.0:1786177380.563456:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306727434 00000100:00000001:2.0:1786177380.563458:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306731270 00000100:00000001:2.0:1786177380.563460:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177380.563462:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177380.563463:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177380.563467:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177380/real 1786177380] req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177380.563475:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177380.563476:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177380.563478:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177380.563480:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177380.563482:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177380.563484:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177380.563487:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177380.563495:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563500:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177380.563501:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.563503:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177380.563504:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177380.563508:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563514:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177380.563517:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563523:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177380.563525:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.563526:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177380.563528:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.563529:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177380.563531:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177380.563534:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177380.563537:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177380.563539:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177380.563543:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177380.563557:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235610496:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177380.563572:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177380.563575:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563582:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177380.563584:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177380.563586:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177380.563587:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177380.563589:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177380.563592:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235610624:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177380.563595:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177380.563597:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177380.563598:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177380.563600:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177380.563604:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9000. 02000000:00000001:2.0:1786177380.563606:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177380.563609:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177380.563612:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177380.563615:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177380.563618:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235610624, portal 4 00000100:00000001:2.0:1786177380.563619:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177380.563621:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487994624 : -110371221556992 : ffff9b9e328bb100) 00000100:00000040:2.0:1786177380.563626:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563632:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177380.563634:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177380.563636:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177380.563638:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235610624, offset 0 00000400:00000200:2.0:1786177380.563642:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177380.563646:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177380.563648:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177380.563651:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177380.563653:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177380.563656:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177380.563658:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177380.563660:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426837:426837:256:4294967295] 192.168.202.132@tcp LPNI seq info [426799:426799:8:4294967295] 00000400:00000200:2.0:1786177380.563664:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177380.563670:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177380.563675:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177380.563678:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177380.563682:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177380.563684:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786177380.563688:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177380.563690:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786177380.563694:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177380.563695:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177380.563698:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177380.563701:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177380.563704:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177380.563706:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177380.563708:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177380.563711:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563718:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177380.563721:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563727:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177380.563729:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177380.563731:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177380.563734:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177380.563737:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177380.563740:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306727434 00000100:00000001:2.0:1786177380.563742:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306731270 00000100:00000001:2.0:1786177380.563745:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306727434 00000100:00000001:2.0:1786177380.563747:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306731270 00000100:00000001:2.0:1786177380.563749:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177380.563751:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177380.563752:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177380.563756:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177380/real 1786177380] req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177380.563763:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177380.563764:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177380.563766:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177380.563768:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177380.563770:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177380.563771:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177380.563775:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177380.563782:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563788:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177380.563790:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.563791:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177380.563793:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177380.563796:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563802:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177380.563805:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563812:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177380.563814:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.563816:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177380.563817:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.563819:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177380.563820:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177380.563823:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177380.563826:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177380.563828:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177380.563832:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177380.563840:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235610624:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177380.563843:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177380.563846:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563852:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177380.563853:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177380.563855:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177380.563857:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177380.563859:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177380.563862:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235610752:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177380.563864:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177380.563866:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177380.563867:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177380.563869:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177380.563872:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177380.563874:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177380.563876:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177380.563879:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177380.563882:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177380.563884:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235610752, portal 4 00000100:00000001:2.0:1786177380.563887:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177380.563889:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487996416 : -110371221555200 : ffff9b9e328bb800) 00000100:00000040:2.0:1786177380.563894:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563901:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177380.563903:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177380.563905:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177380.563908:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235610752, offset 0 00000400:00000200:2.0:1786177380.563910:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177380.563914:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177380.563917:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177380.563920:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177380.563923:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177380.563925:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177380.563927:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177380.563930:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426838:426838:256:4294967295] 192.168.202.132@tcp LPNI seq info [426800:426800:8:4294967295] 00000400:00000200:2.0:1786177380.563934:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177380.563939:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177380.563944:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177380.563947:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177380.563950:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177380.563953:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1786177380.563957:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177380.563960:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1786177380.563963:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177380.563966:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177380.563968:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177380.563972:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177380.563974:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177380.563976:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177380.563978:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177380.563981:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.563995:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177380.563998:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177405 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.564004:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177380.564005:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177380.564008:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177380.564010:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177380.564013:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177380.564016:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306727436 00000100:00000001:2.0:1786177380.564018:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306731270 00000100:00000001:2.0:1786177380.564020:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306727436 00000100:00000001:2.0:1786177380.564022:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306731270 00000100:00000001:2.0:1786177380.564024:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177380.564026:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177380.564028:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177380.564031:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177380/real 1786177380] req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177380.564039:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177380.564040:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177380.564042:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177380.564044:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177380.564046:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177380.564048:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177380.564052:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177380.564059:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.564065:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177380.564067:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.564069:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177380.564070:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177380.564073:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.564080:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177380.564084:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.564090:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177380.564092:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.564094:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177380.564095:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.564097:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177380.564098:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177380.564100:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177380.564103:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177380.564106:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177380.564109:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177380.564117:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235610752:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177380.564122:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.564124:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177380.564127:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9f80 x1872930235610240/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.564133:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177380.564136:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177380.564138:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177380.564141:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177380.564142:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177380.564144:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177380.564147:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177380.564149:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ac800. 02000000:00000001:2.0:1786177380.564152:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177380.564154:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177380.564156:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9f80. 00000100:00000001:2.0:1786177380.564159:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177380.564160:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.564162:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177380.564165:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb480 x1872930235610368/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.564171:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177380.564173:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177380.564174:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177380.564176:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177380.564178:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177380.564179:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177380.564181:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177380.564182:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261acc00. 02000000:00000001:2.0:1786177380.564185:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177380.564186:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177380.564187:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb480. 00000100:00000001:2.0:1786177380.564190:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177380.564191:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.564193:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177380.564196:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328ba680 x1872930235610496/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.564202:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177380.564203:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177380.564205:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786177380.564207:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177380.564208:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177380.564209:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177380.564210:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177380.564212:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261adc00. 02000000:00000001:2.0:1786177380.564213:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177380.564215:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177380.564216:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328ba680. 00000100:00000001:2.0:1786177380.564219:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177380.564220:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.564221:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177380.564225:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb100 x1872930235610624/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.564231:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177380.564232:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177380.564233:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:2.0:1786177380.564235:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177380.564236:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177380.564238:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177380.564239:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177380.564241:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af400. 02000000:00000001:2.0:1786177380.564243:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177380.564244:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177380.564245:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb100. 00000100:00000001:2.0:1786177380.564248:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177380.564249:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.564250:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177380.564254:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bb800 x1872930235610752/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177405 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177380.564261:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177380.564262:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177380.564264:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177380.564266:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177380.564267:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177380.564268:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177380.564270:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177380.564271:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261afc00. 02000000:00000001:2.0:1786177380.564273:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177380.564274:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177380.564276:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bb800. 00000100:00000001:2.0:1786177380.564277:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177380.564278:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.564281:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177380.564287:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177380.564289:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177380.564291:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177380.564292:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177381.083624:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177381.083635:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177381.083638:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177381.083645:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177381.083649:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177381.083653:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177381.083656:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177381.083658:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177381.083662:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177381.452629:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177381.452641:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177381.452655:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177381.452675:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177381.452680:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177381.452685:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177381.452689:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177381.452693:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177381.452695:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177381.452698:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177381.452703:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177381.452705:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177381.452730:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177381.452736:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177381.452740:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177381.452746:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177381.452751:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177381.452756:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177381.452760:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177381.452762:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177381.452766:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177381.452879:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177381.452909:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177381.452913:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177382.092640:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177382.092648:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177382.092650:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177382.092655:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177382.092659:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177382.092663:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177382.092665:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177382.092668:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177382.092671:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177382.476596:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177382.476613:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177382.476622:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177382.476623:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177382.476628:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177382.476629:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177382.476864:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177382.476868:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177382.476871:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177382.477680:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177382.477687:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177382.477689:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177382.477693:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177382.477697:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177382.477700:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177382.477703:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177382.477705:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177382.477708:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177383.116519:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177383.116569:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177383.116572:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177383.116577:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177383.116580:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177383.116584:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177383.116586:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177383.116588:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177383.116591:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177383.502044:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177383.502075:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177383.502078:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177383.502082:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177383.502087:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177383.502089:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177383.502090:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177383.502093:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177383.502099:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177383.502100:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177383.502104:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177383.502106:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177383.502107:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177383.502109:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177383.502111:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177383.502112:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177383.502114:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177383.502116:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177383.502330:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177383.502334:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177383.502336:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177383.504563:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177383.504568:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177383.504570:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177383.504576:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177383.504580:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177383.504583:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177383.504586:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177383.504588:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177383.504591:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177384.140858:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177384.140866:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177384.140868:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177384.140873:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177384.140877:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177384.140880:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177384.140883:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177384.140885:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177384.140888:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177384.460464:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306731270, at 4306731328 00000100:00000040:1.0:1786177384.460464:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306731270, at 4306731328 00000100:00000010:1.0:1786177384.460478:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c120. 00000100:00000010:0.0:1786177384.460479:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c540. 00000100:00000001:1.0:1786177384.524463:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177384.524465:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177384.524469:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177384.524470:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177384.524472:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177384.524474:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177384.524475:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177384.524475:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177384.524478:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177384.524478:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177384.524479:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177384.524482:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177384.524484:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177384.524487:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177384.524490:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177384.524577:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177384.524581:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177384.524584:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177385.164592:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177385.164676:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177385.164679:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177385.164684:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177385.164687:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177385.164690:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177385.164693:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177385.164695:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177385.164698:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177385.548545:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177385.548552:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177385.548558:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177385.548559:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177385.548563:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177385.548564:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177385.548648:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177385.548652:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177385.548655:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177385.549186:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177385.549190:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177385.549192:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177385.549208:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177385.549211:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177385.549215:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177385.549218:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177385.549220:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177385.549222:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177385.676485:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177385.676497:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786177385.676499:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177385.676504:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177385.676509:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177385.676526:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12060 00000400:00000200:0.0:1786177385.676536:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177385.676542:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177385.676545:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177385.676549:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676552:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177385.676555:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676557:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177385.676561:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177385.676563:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786177385.676565:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177385.676568:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786177385.676571:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786177385.676573:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786177385.676576:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676579:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177385.676580:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676582:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177385.676585:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786177385.676589:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676591:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177385.676594:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177385.676598:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177385.676600:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.676603:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177385.676605:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177385.676614:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314380. 00000020:00000040:0.0:1786177385.676618:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786177385.676623:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177385.676625:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.676628:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177385.676631:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177385.676635:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177385.676643:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786177385.676649:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177385.676659:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177385.676668:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177385.676673:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177385.676678:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019d8df06] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177385.676707:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177385.676711:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177385.676714:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786177385.676716:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177385.676718:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177385.676721:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177385.676723:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12060 00000400:00000200:0.0:1786177385.676727:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177385.676729:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177385.676732:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177385.676734:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676737:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177385.676739:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676741:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177385.676743:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177385.676744:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786177385.676746:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177385.676748:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786177385.676750:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786177385.676752:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786177385.676755:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676757:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177385.676759:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676761:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177385.676763:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786177385.676766:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676768:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177385.676770:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177385.676774:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177385.676776:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.676778:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177385.676782:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786177385.676784:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.676786:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177385.676790:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316300. 00000020:00000040:0.0:1786177385.676793:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786177385.676795:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177385.676796:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.676798:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177385.676799:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177385.676801:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177385.676805:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786177385.676808:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177385.676813:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177385.676821:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177385.676823:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177385.676826:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f9ae0ab] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177385.676839:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177385.676842:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177385.676845:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786177385.676847:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177385.676848:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177385.676851:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177385.676853:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12060 00000400:00000200:0.0:1786177385.676856:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177385.676859:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177385.676861:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177385.676863:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676865:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177385.676868:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676869:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177385.676871:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177385.676872:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786177385.676874:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177385.676875:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786177385.676877:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786177385.676879:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786177385.676882:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676884:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177385.676885:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676887:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177385.676889:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786177385.676892:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676894:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177385.676897:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177385.676899:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177385.676901:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.676903:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177385.676905:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786177385.676907:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.676909:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177385.676912:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317800. 00000020:00000040:0.0:1786177385.676914:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786177385.676916:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177385.676917:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.676918:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177385.676920:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177385.676922:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177385.676925:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786177385.676928:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177385.676932:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177385.676939:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177385.676941:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177385.676944:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bc84893] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177385.676951:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177385.676953:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177385.676956:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786177385.676957:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177385.676970:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177385.676972:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177385.676975:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12060 00000400:00000200:0.0:1786177385.676978:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177385.676981:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177385.676983:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177385.676986:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676988:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177385.676990:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.676992:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177385.676994:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177385.676996:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786177385.676997:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177385.676999:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786177385.677001:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786177385.677003:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786177385.677007:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.677008:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177385.677010:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.677012:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177385.677014:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786177385.677017:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177385.677019:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177385.677021:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177385.677024:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177385.677025:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.677027:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177385.677029:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786177385.677032:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.677033:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177385.677037:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316a00. 00000020:00000040:0.0:1786177385.677039:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177385.677041:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177385.677043:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.677044:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177385.677046:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177385.677047:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177385.677050:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786177385.677054:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177385.677058:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177385.677065:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177385.677067:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177385.677070:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000968a00e5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177385.677077:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177385.677080:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177385.677083:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786177385.677084:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177385.677085:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177385.677088:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177385.677090:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12060 00000400:00000200:0.0:1786177385.677093:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177385.677096:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177385.677098:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177385.677100:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.677102:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177385.677104:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.677106:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177385.677108:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177385.677109:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786177385.677111:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177385.677113:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786177385.677115:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786177385.677117:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786177385.677120:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.677121:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177385.677124:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177385.677125:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177385.677128:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786177385.677130:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177385.677132:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177385.677134:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177385.677136:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177385.677138:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.677139:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177385.677141:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786177385.677143:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.677144:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177385.677147:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315c00. 00000020:00000040:0.0:1786177385.677149:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177385.677152:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177385.677153:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177385.677154:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177385.677155:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177385.677157:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177385.677160:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786177385.677163:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177385.677167:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177385.677174:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177385.677176:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177385.677180:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f38b1e8d] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177385.677187:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177385.677191:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12070) 00000100:00000001:2.0:1786177385.677308:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177385.677312:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177385.677314:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177385.677321:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677329:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177385.677331:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177385.677335:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177385.677337:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177385.677339:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177385.677342:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235610880:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177385.677346:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177385.677350:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177385.677352:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177385.677355:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177385.677383:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177385.677386:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177385.677391:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177385.677395:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177385.677401:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177385.677407:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235610880, portal 25 00000100:00000001:2.0:1786177385.677410:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177385.677411:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794202496 : -110371915349120 : ffff9b9e09314380) 00000100:00000040:2.0:1786177385.677416:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677423:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177385.677426:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177385.677428:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177385.677431:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235610880, offset 0 00000400:00000200:2.0:1786177385.677438:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177385.677445:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177385.677451:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177385.677455:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177385.677459:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177385.677462:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177385.677465:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177385.677469:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426839:426839:256:4294967295] 192.168.202.132@tcp LPNI seq info [426801:426801:8:4294967295] 00000400:00000200:2.0:1786177385.677473:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177385.677482:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177385.677488:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177385.677492:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177385.677499:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177385.677502:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177385.677507:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177385.677510:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177385.677526:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177385.677529:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177385.677535:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177385.677539:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177385.677543:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177385.677545:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177385.677549:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177385.677553:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677567:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177385.677570:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677576:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177385.677578:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177385.677583:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177385.677586:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177385.677589:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306736385 00000100:00000010:2.0:1786177385.677594:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c780. 00000100:00000001:2.0:1786177385.677600:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177385.677605:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306736385 00000100:00000010:2.0:1786177385.677607:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c600. 00000100:00000001:2.0:1786177385.677610:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177385.677613:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177385.677616:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177385.677618:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177385.677622:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177385/real 1786177385] req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177385.677631:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177385.677633:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177385.677635:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177385.677638:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177385.677639:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177385.677641:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177385.677644:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177385.677652:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677658:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177385.677660:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.677662:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177385.677664:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177385.677667:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677674:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177385.677678:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677684:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177385.677687:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.677688:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177385.677689:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.677691:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177385.677693:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177385.677696:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177385.677700:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177385.677702:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177385.677706:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177385.677713:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235610880:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177385.677717:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177385.677720:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677726:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177385.677727:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177385.677729:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177385.677731:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177385.677733:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177385.677735:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611008:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177385.677739:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177385.677740:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177385.677742:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177385.677744:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177385.677746:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9000. 02000000:00000001:2.0:1786177385.677748:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177385.677752:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177385.677755:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177385.677758:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177385.677760:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235611008, portal 10 00000100:00000001:2.0:1786177385.677762:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177385.677763:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794210560 : -110371915341056 : ffff9b9e09316300) 00000100:00000040:2.0:1786177385.677770:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677776:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177385.677778:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177385.677781:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177385.677783:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235611008, offset 0 00000400:00000200:2.0:1786177385.677786:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177385.677790:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177385.677793:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177385.677796:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177385.677798:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177385.677801:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177385.677803:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177385.677806:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426840:426840:256:4294967295] 192.168.202.132@tcp LPNI seq info [426802:426802:8:4294967295] 00000400:00000200:2.0:1786177385.677810:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177385.677816:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177385.677820:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177385.677824:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177385.677827:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177385.677831:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177385.677835:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177385.677837:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177385.677840:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177385.677842:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177385.677844:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177385.677847:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177385.677850:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177385.677852:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177385.677854:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177385.677857:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677864:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177385.677867:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677873:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177385.677875:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177385.677878:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177385.677881:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177385.677885:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177385.677887:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306732548 00000100:00000001:2.0:1786177385.677889:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306736385 00000100:00000001:2.0:1786177385.677891:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306732548 00000100:00000001:2.0:1786177385.677893:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306736385 00000100:00000001:2.0:1786177385.677895:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177385.677898:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177385.677899:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177385.677903:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177385/real 1786177385] req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177385.677910:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177385.677911:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177385.677913:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177385.677916:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177385.677917:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177385.677919:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177385.677922:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177385.677930:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677938:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177385.677939:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.677941:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177385.677942:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177385.677946:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.677953:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177385.677956:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.678587:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177385.678590:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.678592:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177385.678594:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.678596:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177385.678598:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177385.678601:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177385.678605:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177385.678608:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177385.678613:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177385.678623:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611008:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177385.678628:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177385.678631:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.678638:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177385.678639:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177385.678642:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177385.678643:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177385.678645:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177385.678648:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611136:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177385.678651:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177385.678654:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177385.678655:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177385.678657:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177385.678661:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786177385.678663:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177385.678667:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177385.678669:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177385.678673:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177385.678676:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235611136, portal 10 00000100:00000001:2.0:1786177385.678678:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177385.678679:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215936 : -110371915335680 : ffff9b9e09317800) 00000100:00000040:2.0:1786177385.678685:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.678692:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177385.678694:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177385.678696:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177385.678699:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235611136, offset 0 00000400:00000200:2.0:1786177385.678703:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177385.678707:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177385.678711:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177385.678714:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177385.678717:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177385.678719:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177385.678721:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177385.678724:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426841:426841:256:4294967295] 192.168.202.132@tcp LPNI seq info [426803:426803:8:4294967295] 00000400:00000200:2.0:1786177385.678729:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177385.678736:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177385.678740:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177385.678743:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177385.678747:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177385.678750:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177385.678755:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177385.678757:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177385.678761:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177385.678763:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177385.678766:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177385.678769:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177385.678772:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177385.678774:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177385.678776:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177385.678779:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.678787:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177385.678790:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.678797:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177385.678798:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177385.678801:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177385.678804:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177385.678807:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177385.678810:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306732549 00000100:00000001:2.0:1786177385.678812:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306736385 00000100:00000001:2.0:1786177385.678814:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306732549 00000100:00000001:2.0:1786177385.678817:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306736385 00000100:00000001:2.0:1786177385.678819:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177385.678821:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177385.678823:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177385.678827:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177385/real 1786177385] req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177385.678834:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177385.678835:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177385.678837:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177385.678839:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177385.678841:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177385.678843:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177385.678846:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177385.678854:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.678861:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177385.678862:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.678864:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177385.678866:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177385.678869:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.678875:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177385.678879:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.678886:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177385.678888:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.678890:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177385.678891:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.678893:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177385.678895:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177385.678898:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177385.678901:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177385.678903:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177385.678907:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177385.678915:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611136:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177385.678919:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177385.678922:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.678929:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177385.678930:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177385.678932:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177385.678934:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177385.678936:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177385.678938:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611264:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177385.678941:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177385.678943:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177385.678945:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177385.678946:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177385.678949:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177385.678951:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177385.678954:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177385.678956:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177385.678959:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177385.678962:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235611264, portal 4 00000100:00000001:2.0:1786177385.678963:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177385.678964:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794212352 : -110371915339264 : ffff9b9e09316a00) 00000100:00000040:2.0:1786177385.678970:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.678976:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177385.678990:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177385.678992:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177385.678995:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235611264, offset 0 00000400:00000200:2.0:1786177385.678999:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177385.679002:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177385.679005:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177385.679008:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177385.679010:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177385.679012:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177385.679014:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177385.679017:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426842:426842:256:4294967295] 192.168.202.132@tcp LPNI seq info [426804:426804:8:4294967295] 00000400:00000200:2.0:1786177385.679021:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177385.679028:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177385.679032:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177385.679035:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177385.679038:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177385.679041:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177385.679045:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177385.679047:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177385.679051:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177385.679053:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177385.679056:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177385.679059:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177385.679061:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177385.679063:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177385.679065:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177385.679069:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679076:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177385.679079:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679091:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177385.679092:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177385.679094:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177385.679097:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177385.679100:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177385.679102:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306732549 00000100:00000001:2.0:1786177385.679104:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306736385 00000100:00000001:2.0:1786177385.679106:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306732549 00000100:00000001:2.0:1786177385.679107:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306736385 00000100:00000001:2.0:1786177385.679109:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177385.679111:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177385.679113:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177385.679117:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177385/real 1786177385] req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177385.679125:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177385.679126:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177385.679127:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177385.679129:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177385.679131:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177385.679133:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177385.679136:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177385.679144:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679151:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177385.679152:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679154:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177385.679155:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177385.679158:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679165:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177385.679169:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679175:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177385.679177:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679179:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177385.679180:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679182:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177385.679183:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177385.679186:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177385.679188:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177385.679190:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177385.679194:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177385.679211:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611264:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177385.679215:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177385.679218:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679226:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177385.679227:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177385.679229:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177385.679230:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177385.679232:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177385.679235:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611392:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177385.679243:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177385.679245:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177385.679247:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177385.679249:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177385.679252:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177385.679253:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177385.679256:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177385.679258:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177385.679261:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177385.679264:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235611392, portal 4 00000100:00000001:2.0:1786177385.679265:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177385.679267:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794208768 : -110371915342848 : ffff9b9e09315c00) 00000100:00000040:2.0:1786177385.679272:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679278:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177385.679281:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177385.679282:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177385.679284:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235611392, offset 0 00000400:00000200:2.0:1786177385.679287:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177385.679291:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177385.679294:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177385.679297:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177385.679299:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177385.679301:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177385.679303:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177385.679306:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426843:426843:256:4294967295] 192.168.202.132@tcp LPNI seq info [426805:426805:8:4294967295] 00000400:00000200:2.0:1786177385.679310:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177385.679316:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177385.679320:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177385.679322:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177385.679326:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177385.679329:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1786177385.679332:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177385.679335:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1786177385.679339:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177385.679341:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177385.679343:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177385.679346:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177385.679349:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177385.679351:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177385.679353:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177385.679356:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679379:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177385.679382:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177410 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679388:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177385.679390:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177385.679393:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177385.679395:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177385.679398:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177385.679400:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306732552 00000100:00000001:2.0:1786177385.679402:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306736385 00000100:00000001:2.0:1786177385.679405:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306732552 00000100:00000001:2.0:1786177385.679407:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306736385 00000100:00000001:2.0:1786177385.679409:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177385.679411:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177385.679412:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177385.679415:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177385/real 1786177385] req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177385.679422:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177385.679423:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177385.679425:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177385.679427:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177385.679429:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177385.679431:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177385.679434:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177385.679442:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679448:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177385.679449:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679451:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177385.679452:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177385.679455:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679462:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177385.679466:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679472:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177385.679474:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679476:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177385.679477:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679478:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177385.679480:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177385.679482:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177385.679486:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177385.679488:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177385.679491:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177385.679500:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09315c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611392:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177385.679504:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679506:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177385.679509:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314380 x1872930235610880/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679527:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177385.679529:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177385.679532:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:2.0:1786177385.679535:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177385.679537:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177385.679538:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177385.679540:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177385.679543:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:2.0:1786177385.679546:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177385.679548:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177385.679549:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314380. 00000100:00000001:2.0:1786177385.679552:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177385.679554:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679555:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177385.679559:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316300 x1872930235611008/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679565:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177385.679566:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177385.679568:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:2.0:1786177385.679570:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177385.679571:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177385.679572:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177385.679574:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177385.679575:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:2.0:1786177385.679577:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177385.679578:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177385.679580:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316300. 00000100:00000001:2.0:1786177385.679582:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177385.679583:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679584:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177385.679587:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317800 x1872930235611136/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679593:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177385.679594:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177385.679596:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:2.0:1786177385.679598:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177385.679599:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177385.679600:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177385.679602:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177385.679603:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:2.0:1786177385.679605:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177385.679606:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177385.679607:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317800. 00000100:00000001:2.0:1786177385.679609:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177385.679610:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679611:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177385.679614:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316a00 x1872930235611264/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679619:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177385.679621:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177385.679622:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:2.0:1786177385.679624:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177385.679625:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177385.679626:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177385.679627:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177385.679629:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:2.0:1786177385.679630:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177385.679632:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177385.679633:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316a00. 00000100:00000001:2.0:1786177385.679635:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177385.679636:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679637:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177385.679640:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09315c00 x1872930235611392/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177410 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177385.679645:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177385.679646:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177385.679647:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:2.0:1786177385.679649:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177385.679650:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177385.679652:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177385.679654:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177385.679655:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:2.0:1786177385.679657:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177385.679659:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177385.679660:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09315c00. 00000100:00000001:2.0:1786177385.679662:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177385.679663:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679665:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177385.679671:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177385.679673:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177385.679675:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177385.679676:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177386.188485:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177386.188493:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177386.188496:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177386.188502:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177386.188505:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177386.188519:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177386.188522:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177386.188525:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177386.188529:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177386.572615:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177386.572623:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177386.572625:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177386.572642:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177386.572646:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177386.572650:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177386.572653:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177386.572656:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177386.572659:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177386.572803:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177386.572806:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177386.572809:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177386.572826:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177386.572830:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177386.572831:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177386.572930:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177386.573008:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177386.573013:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177387.212754:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177387.212762:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177387.212764:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177387.212771:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177387.212775:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177387.212778:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177387.212781:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177387.212783:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177387.212787:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177387.597469:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177387.597475:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177387.597481:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177387.597483:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177387.597486:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177387.597488:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177387.597915:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177387.597919:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177387.597923:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177387.597993:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177387.597997:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177387.598000:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177387.598004:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177387.598007:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177387.598010:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177387.598014:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177387.598017:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177387.598019:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177388.236838:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177388.236846:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177388.236848:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177388.236853:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177388.236857:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177388.236861:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177388.236863:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177388.236866:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177388.236868:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177388.620495:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177388.620502:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177388.620504:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177388.620509:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177388.620512:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177388.620516:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177388.620519:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177388.620521:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177388.620525:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177388.620615:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177388.620618:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177388.620622:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177388.620623:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177388.620637:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177388.620638:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177388.620757:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177388.620760:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177388.620763:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.0:1786177389.196445:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:0.0:1786177389.196454:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:0.0:1786177389.196459:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:1.0:1786177389.199533:0:2386:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1786177389.199545:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1786177389.199587:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1786177389.199592:0:2386:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:1.0:1786177389.199595:0:2386:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177389.261785:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177389.261794:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177389.261796:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177389.262178:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177389.262183:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177389.262187:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177389.262190:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177389.262192:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177389.262195:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177389.580469:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306736385, at 4306736448 00000100:00000010:1.0:1786177389.580494:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c600. 00000100:00000040:0.0:1786177389.580601:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306736385, at 4306736448 00000100:00000010:0.0:1786177389.580607:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c780. 00000100:00000001:1.0:1786177389.644654:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177389.644661:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177389.644663:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177389.644669:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177389.644673:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177389.644676:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177389.644679:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177389.644682:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177389.644685:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177389.645121:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177389.645125:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177389.645128:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177389.645130:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177389.645133:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177389.645134:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177389.645341:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177389.645345:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177389.645348:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1786177390.284490:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177390.284499:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177390.284501:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177390.284506:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177390.284509:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177390.284513:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177390.284515:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177390.284518:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177390.284521:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177390.668495:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177390.668501:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177390.668508:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177390.668509:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177390.668512:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177390.668513:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177390.668617:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177390.668622:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177390.668625:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177390.668730:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177390.668734:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177390.668736:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177390.668740:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177390.668744:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177390.668747:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177390.668750:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177390.668753:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177390.668756:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177390.796457:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177390.796476:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177390.796479:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177390.796483:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177390.796487:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177390.796495:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12065 00000400:00000200:2.0:1786177390.796503:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177390.796509:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177390.796512:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177390.796515:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.796519:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177390.796521:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.796523:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177390.796527:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177390.796529:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177390.796532:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177390.796535:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177390.796537:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177390.796539:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177390.796542:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.796544:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177390.796546:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.796548:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177390.796550:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177390.796553:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177390.796556:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177390.796559:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177390.796562:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177390.796565:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.796567:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177390.796569:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177390.796577:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735500. 00000020:00000040:2.0:1786177390.796586:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177390.796590:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177390.796592:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.796709:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177390.796713:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177390.796717:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177390.796725:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9c00. 00000100:00000001:2.0:1786177390.796732:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177390.796742:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177390.796753:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177390.796758:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177390.796763:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000abb79f7f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.796878:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177390.796882:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177390.796884:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177390.796891:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.796904:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177390.796907:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177390.796910:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177390.796913:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177390.796916:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177390.796919:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235611520:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177390.796923:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177390.796926:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177390.796929:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177390.796932:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177390.796938:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177390.796940:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177390.796945:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177390.796949:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177390.796956:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177390.796963:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235611520, portal 25 00000100:00000001:0.0:1786177390.796965:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177390.796967:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620992 : -110371239930624 : ffff9b9e31735500) 00000100:00000040:0.0:1786177390.796973:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.796979:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177390.796982:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177390.796985:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177390.796988:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235611520, offset 0 00000400:00000200:0.0:1786177390.796994:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177390.797002:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177390.797009:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177390.797014:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177390.797019:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177390.797034:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177390.797036:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177390.797040:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426844:426844:256:4294967295] 192.168.202.132@tcp LPNI seq info [426806:426806:8:4294967295] 00000400:00000200:0.0:1786177390.797045:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177390.797055:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177390.797061:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177390.797067:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177390.797083:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177390.797087:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1786177390.797093:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177390.797096:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1786177390.797100:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177390.797102:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177390.797109:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177390.797112:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177390.797118:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177390.797120:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177390.797124:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177390.797127:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177390.798750:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177390.798757:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177390.798761:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177390.798762:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177390.798764:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177390.798778:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177390.798783:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12065 00000400:00000200:2.0:1786177390.798789:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177390.798794:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177390.798796:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177390.798799:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.798802:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177390.798804:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.798805:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177390.798808:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177390.798810:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177390.798812:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177390.798815:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177390.798817:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177390.798819:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177390.798822:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.798823:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177390.798825:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.798827:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177390.798829:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177390.798831:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177390.798833:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177390.798835:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177390.798842:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177390.798845:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.798846:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177390.798850:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177390.798853:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.798854:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177390.798861:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737b80. 00000020:00000040:2.0:1786177390.798864:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177390.798868:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177390.798870:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.798872:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177390.798874:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177390.798877:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177390.798884:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8400. 00000100:00000001:2.0:1786177390.798890:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177390.798897:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177390.798906:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177390.798910:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177390.798913:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bdd02bd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.799387:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177390.799394:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.799402:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.799404:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177390.799415:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177390.799420:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177390.799423:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306741507 00000100:00000010:0.0:1786177390.799428:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5ae0. 00000100:00000001:0.0:1786177390.799434:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177390.799441:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306741507 00000100:00000010:0.0:1786177390.799444:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5c60. 00000100:00000001:0.0:1786177390.799447:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177390.799450:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177390.799453:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.799456:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177390.799460:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177390/real 1786177390] req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177390.799482:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177390.799484:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177390.799487:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177390.799490:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177390.799492:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177390.799495:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177390.799499:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177390.799508:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.799514:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177390.799516:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.799518:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177390.799521:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177390.799524:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.799531:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177390.799535:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.799543:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177390.799545:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.799547:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177390.799548:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.799551:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177390.799552:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177390.799555:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177390.799559:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177390.799562:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177390.799566:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177390.799573:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235611520:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177390.799578:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.799581:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177390.799584:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735500 x1872930235611520/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.799590:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177390.799592:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177390.799732:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177390.799738:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177390.799740:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177390.799742:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177390.799745:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177390.799747:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9c00. 02000000:00000001:0.0:1786177390.799751:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177390.799753:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177390.799755:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735500. 00000100:00000001:0.0:1786177390.799758:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177390.799759:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.799762:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.799779:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177390.799781:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.799784:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177390.799786:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177390.799788:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177390.799793:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.799800:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177390.799802:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177390.799805:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177390.799808:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177390.799810:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177390.799813:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611648:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177390.799816:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177390.799820:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177390.799822:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177390.799825:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177390.799829:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177390.799830:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177390.799835:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177390.799838:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177390.799842:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177390.799848:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235611648, portal 10 00000100:00000001:0.0:1786177390.799850:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177390.799852:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469630848 : -110371239920768 : ffff9b9e31737b80) 00000100:00000040:0.0:1786177390.799859:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.799866:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177390.799869:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177390.799871:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177390.799873:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235611648, offset 0 00000400:00000200:0.0:1786177390.799878:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177390.799884:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177390.799889:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177390.799893:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177390.799897:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177390.799900:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177390.799902:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177390.799907:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426845:426845:256:4294967295] 192.168.202.132@tcp LPNI seq info [426807:426807:8:4294967295] 00000400:00000200:0.0:1786177390.799911:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177390.799919:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177390.799925:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177390.799930:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177390.799935:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177390.799939:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1786177390.799944:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177390.799948:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1786177390.799987:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177390.799990:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177390.799993:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177390.799997:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177390.800001:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177390.800003:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177390.800006:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177390.800010:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.800017:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177390.800020:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.800129:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.800131:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177390.800135:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177390.800139:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177390.800142:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177390.800202:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306737670 00000100:00000001:0.0:1786177390.800206:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306741507 00000100:00000001:0.0:1786177390.800208:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306737670 00000100:00000001:0.0:1786177390.800210:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306741507 00000100:00000001:0.0:1786177390.800212:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177390.800215:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.800217:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177390.800222:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177390/real 1786177390] req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177390.800235:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177390.800237:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177390.800239:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177390.800241:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177390.800243:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177390.800245:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177390.800248:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177390.800255:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.800261:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177390.800263:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.800264:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177390.800266:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177390.800270:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.800277:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177390.800280:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.800287:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177390.800289:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.800291:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177390.800292:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.800294:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177390.800296:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177390.800299:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177390.800302:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177390.800304:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177390.800308:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177390.800315:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611648:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177390.800319:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.800321:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177390.800324:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737b80 x1872930235611648/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.800330:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177390.800331:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177390.800333:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177390.800335:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177390.800336:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177390.800338:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177390.800339:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177390.800341:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8400. 02000000:00000001:0.0:1786177390.800343:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177390.800344:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177390.800345:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737b80. 00000100:00000001:0.0:1786177390.800347:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177390.800348:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.800350:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.800353:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177390.800354:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.800356:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177390.800379:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.800506:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177390.800510:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177390.800513:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177390.800514:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177390.800516:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177390.800519:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177390.800523:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12065 00000400:00000200:2.0:1786177390.800527:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177390.800530:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177390.800532:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177390.800535:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.800538:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177390.800540:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.800542:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177390.800544:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177390.800546:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177390.800548:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177390.800550:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177390.800552:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177390.800554:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177390.800557:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.800559:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177390.800560:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.800562:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177390.800565:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177390.800567:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177390.800570:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177390.800572:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177390.800576:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177390.800578:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.800580:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177390.800583:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177390.800585:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.800587:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177390.800592:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734380. 00000020:00000040:2.0:1786177390.800594:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000100:00000001:0.0:1786177390.800732:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177390.800735:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177390.800762:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177390.800764:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.800827:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177390.800830:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177390.800834:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177390.800840:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aabc00. 00000100:00000001:2.0:1786177390.800845:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177390.800852:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177390.800861:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177390.800864:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177390.800868:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fbadf61] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.800911:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177390.800913:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177390.800915:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177390.800919:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.800926:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177390.800928:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177390.800930:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177390.800932:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177390.800934:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177390.800937:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611776:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177390.800941:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177390.800943:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177390.800945:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177390.800947:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177390.800950:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177390.800952:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177390.800956:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177390.800959:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177390.800963:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177390.800967:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235611776, portal 10 00000100:00000001:0.0:1786177390.800969:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177390.800970:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469616512 : -110371239935104 : ffff9b9e31734380) 00000100:00000040:0.0:1786177390.800976:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.800981:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177390.800984:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177390.800986:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177390.800988:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235611776, offset 0 00000400:00000200:0.0:1786177390.800992:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177390.800997:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177390.801001:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177390.801005:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177390.801008:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177390.801010:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177390.801013:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177390.801016:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426846:426846:256:4294967295] 192.168.202.132@tcp LPNI seq info [426808:426808:8:4294967295] 00000400:00000200:0.0:1786177390.801021:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177390.801207:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177390.801210:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177390.801214:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177390.801215:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177390.801217:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177390.801219:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177390.801222:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12065 00000400:00000200:2.0:1786177390.801226:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177390.801229:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177390.801231:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177390.801234:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.801235:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177390.801237:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.801239:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177390.801241:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177390.801242:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177390.801243:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177390.801245:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177390.801247:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177390.801249:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177390.801252:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.801254:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177390.801256:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.801257:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177390.801259:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177390.801261:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177390.801263:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177390.801264:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177390.801268:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177390.801269:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.801271:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177390.801273:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177390.801275:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.801277:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177390.801280:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736300. 00000020:00000040:2.0:1786177390.801282:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177390.801284:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177390.801286:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.801287:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177390.801289:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177390.801290:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177390.801294:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9000. 00000100:00000001:2.0:1786177390.801298:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177390.801303:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177390.801309:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177390.801311:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177390.801314:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004426a6d9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177390.801320:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177390.801322:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177390.801325:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177390.801326:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177390.801327:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177390.801329:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177390.801331:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12065 00000400:00000200:2.0:1786177390.801334:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177390.801336:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177390.801338:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177390.801340:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.801342:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177390.801343:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.801345:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177390.801346:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177390.801347:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177390.801348:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177390.801350:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177390.801351:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177390.801353:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177390.801372:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.801375:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177390.801377:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177390.801379:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177390.801381:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177390.801384:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177390.801386:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177390.801387:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177390.801390:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177390.801391:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.801393:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177390.801394:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177390.801396:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.801398:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177390.801401:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734000. 00000020:00000040:2.0:1786177390.801403:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177390.801405:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177390.801407:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177390.801408:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177390.801410:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177390.801411:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177390.801415:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa800. 00000100:00000001:2.0:1786177390.801418:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177390.801422:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177390.801428:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177390.801430:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177390.801432:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f47f6dd2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177390.801439:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177390.801443:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12075) 00000400:00000200:0.0:1786177390.801511:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177390.801541:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177390.801545:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177390.801550:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177390.801553:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1786177390.801558:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177390.801560:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1786177390.801563:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177390.801565:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177390.801568:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177390.801572:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177390.801575:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177390.801577:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177390.801579:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177390.801583:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.801591:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177390.801594:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.801703:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.801706:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177390.801711:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177390.801715:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177390.801719:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177390.801721:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306737672 00000100:00000001:0.0:1786177390.801724:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306741507 00000100:00000001:0.0:1786177390.801726:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306737672 00000100:00000001:0.0:1786177390.801728:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306741507 00000100:00000001:0.0:1786177390.801730:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177390.801732:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.801734:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177390.801738:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177390/real 1786177390] req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177390.801747:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177390.801749:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177390.801750:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177390.801753:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177390.801754:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177390.801756:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177390.801760:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177390.801767:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.801836:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177390.801839:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.801841:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177390.801842:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177390.801846:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.801851:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177390.801855:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.801861:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177390.801863:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.801864:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177390.801865:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.801868:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177390.801870:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177390.801872:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177390.801875:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177390.801877:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177390.801896:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177390.801904:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611776:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177390.801907:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.801909:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177390.801911:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734380 x1872930235611776/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.801917:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177390.801919:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177390.801921:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177390.801923:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177390.801925:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177390.801926:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177390.801928:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177390.801929:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aabc00. 02000000:00000001:0.0:1786177390.801932:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177390.801933:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177390.801934:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734380. 00000100:00000001:0.0:1786177390.801936:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177390.801938:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.801939:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.801944:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177390.801945:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.801947:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177390.801948:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177390.801950:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177390.801953:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.801958:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177390.801959:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177390.801961:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177390.801963:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177390.801964:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177390.801967:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611904:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177390.801971:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177390.801973:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177390.801975:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177390.801976:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177390.801980:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177390.801981:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177390.801986:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177390.801988:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177390.801992:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177390.801995:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235611904, portal 4 00000100:00000001:0.0:1786177390.801996:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177390.801998:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469624576 : -110371239927040 : ffff9b9e31736300) 00000100:00000040:0.0:1786177390.802004:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802011:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177390.802013:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177390.802015:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177390.802017:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235611904, offset 0 00000400:00000200:0.0:1786177390.802021:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177390.802024:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177390.802198:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177390.802203:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177390.802206:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177390.802209:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177390.802211:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177390.802215:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426847:426847:256:4294967295] 192.168.202.132@tcp LPNI seq info [426809:426809:8:4294967295] 00000400:00000200:0.0:1786177390.802219:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177390.802225:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177390.802229:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177390.802234:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177390.802238:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177390.802240:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1786177390.802244:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177390.802246:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1786177390.802249:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177390.802251:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177390.802254:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177390.802257:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177390.802260:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177390.802261:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177390.802263:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177390.802267:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802274:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177390.802277:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802284:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.802285:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177390.802289:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177390.802292:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177390.802295:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177390.802297:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306737672 00000100:00000001:0.0:1786177390.802299:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306741507 00000100:00000001:0.0:1786177390.802301:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306737672 00000100:00000001:0.0:1786177390.802303:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306741507 00000100:00000001:0.0:1786177390.802305:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177390.802307:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.802309:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177390.802313:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177390/real 1786177390] req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177390.802321:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177390.802322:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177390.802324:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177390.802326:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177390.802328:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177390.802330:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177390.802333:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177390.802341:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802348:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177390.802349:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.802350:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177390.802352:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177390.802355:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802381:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177390.802385:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802391:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177390.802394:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.802395:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177390.802396:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.802398:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177390.802399:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177390.802402:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177390.802405:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177390.802406:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177390.802410:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177390.802423:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235611904:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177390.802427:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177390.802430:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802436:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177390.802437:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177390.802439:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177390.802440:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177390.802442:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177390.802445:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612032:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177390.802448:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177390.802450:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177390.802452:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177390.802454:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177390.802457:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177390.802459:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177390.802474:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177390.802479:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177390.802482:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177390.802485:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235612032, portal 4 00000100:00000001:0.0:1786177390.802487:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177390.802488:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469615616 : -110371239936000 : ffff9b9e31734000) 00000100:00000040:0.0:1786177390.802493:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802499:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177390.802501:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177390.802503:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177390.802505:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235612032, offset 0 00000400:00000200:0.0:1786177390.802509:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177390.802514:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177390.802517:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177390.802520:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177390.802522:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177390.802524:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177390.802526:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177390.802529:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426848:426848:256:4294967295] 192.168.202.132@tcp LPNI seq info [426810:426810:8:4294967295] 00000400:00000200:0.0:1786177390.802532:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177390.802538:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177390.802543:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177390.802546:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177390.802550:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177390.802553:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1786177390.802557:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177390.802559:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 42 secs later) 00000800:00000100:0.0:1786177390.802562:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177390.802564:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177390.802567:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177390.802570:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177390.802573:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177390.802575:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177390.802577:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177390.802581:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802588:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177390.802590:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177415 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802595:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.802707:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177390.802713:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177390.802716:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177390.802719:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177390.802721:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306737675 00000100:00000001:0.0:1786177390.802723:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306741507 00000100:00000001:0.0:1786177390.802725:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306737675 00000100:00000001:0.0:1786177390.802727:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306741507 00000100:00000001:0.0:1786177390.802729:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177390.802731:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.802733:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177390.802737:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177390/real 1786177390] req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177390.802745:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177390.802746:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177390.802747:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177390.802750:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177390.802751:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177390.802754:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177390.802756:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177390.802765:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802835:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177390.802836:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.802838:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177390.802839:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177390.802844:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802850:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177390.802853:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802860:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177390.802862:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.802863:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177390.802865:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.802866:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177390.802868:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177390.802870:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177390.802873:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177390.802875:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177390.802878:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177390.802885:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612032:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177390.802889:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.802890:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177390.802893:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736300 x1872930235611904/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802899:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177390.802901:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177390.802903:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177390.802906:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177390.802907:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177390.802909:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177390.802911:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177390.802913:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9000. 02000000:00000001:0.0:1786177390.802915:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177390.802916:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177390.802917:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736300. 00000100:00000001:0.0:1786177390.802920:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177390.802921:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.802923:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177390.802926:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734000 x1872930235612032/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177415 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177390.802936:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177390.802937:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177390.802938:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177390.802940:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177390.802941:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177390.802942:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177390.802944:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177390.802945:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa800. 02000000:00000001:0.0:1786177390.802946:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177390.802947:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177390.802948:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734000. 00000100:00000001:0.0:1786177390.802950:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177390.802951:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.802953:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177390.802957:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177390.802959:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177390.802961:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177390.802962:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177391.309799:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177391.309877:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177391.309879:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177391.309885:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177391.309889:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177391.309892:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177391.309894:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177391.309897:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177391.309899:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177391.692426:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177391.692434:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177391.692437:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177391.692442:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177391.692445:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177391.692448:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177391.692450:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177391.692463:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177391.692467:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177391.692541:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177391.692544:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177391.692548:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177391.692550:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177391.692552:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177391.692556:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177391.692559:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177391.692561:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177391.692563:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177391.692565:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177391.692568:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177391.692570:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177391.693162:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177391.693167:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177391.693171:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177392.544116:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177392.544123:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177392.544125:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177392.544130:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177392.544134:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177392.544137:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177392.544140:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177392.544142:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177392.544145:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177392.716481:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177392.716487:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177392.716489:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177392.716492:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177392.716496:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177392.716497:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177392.716500:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177392.716501:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177392.716506:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177392.716506:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177392.716510:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177392.716512:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177392.716517:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177392.716519:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177392.716522:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177392.716612:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177392.716616:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177392.716618:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177393.549178:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177393.549184:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177393.549187:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177393.549192:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177393.549195:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177393.549199:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177393.549202:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177393.549205:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177393.549209:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177393.740460:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177393.740468:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177393.740471:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177393.740483:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177393.740488:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177393.740495:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177393.740498:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177393.740500:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177393.740503:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1786177393.740506:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177393.740509:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177393.740510:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177393.740514:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177393.740516:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177393.740519:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:1.0:1786177393.740520:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177393.740521:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177393.740523:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177393.740526:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177393.740527:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177393.740531:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177393.740532:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177393.740533:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177393.740538:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177393.740538:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177393.740542:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177393.740547:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177393.740605:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177393.740609:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177393.740612:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177394.572521:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177394.572529:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177394.572532:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177394.572537:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177394.572541:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177394.572545:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177394.572547:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177394.572550:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177394.572554:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177394.700496:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306741507, at 4306741568 00000100:00000010:1.0:1786177394.700512:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5c60. 00000100:00000040:0.0:1786177394.700530:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306741507, at 4306741568 00000100:00000010:0.0:1786177394.700547:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5ae0. 00000100:00000001:1.0:1786177394.764551:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177394.764560:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177394.764563:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177394.764567:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1786177394.764570:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00010000:00000001:0.0:1786177394.764575:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177394.764577:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177394.764587:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177394.764587:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177394.764590:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177394.764600:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177394.764602:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177394.764605:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177394.764606:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177394.764612:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177394.769485:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177394.769496:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177394.769502:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177395.596467:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177395.596474:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177395.596477:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177395.596483:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177395.596486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177395.596490:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177395.596493:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177395.596495:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177395.596498:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177395.788536:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177395.788543:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1786177395.788546:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177395.788546:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177395.788554:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177395.788556:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177395.788565:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177395.788567:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177395.788569:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177395.788571:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177395.788574:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1786177395.788578:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177395.788579:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177395.788583:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177395.788586:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177395.788690:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177395.788695:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177395.788699:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1786177395.920679:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177395.920694:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786177395.920698:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177395.920702:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177395.920708:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177395.920717:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12070 00000400:00000200:0.0:1786177395.920728:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177395.920735:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177395.920739:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177395.920742:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.920745:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177395.920748:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.920751:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177395.920757:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177395.920759:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786177395.920762:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177395.920766:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786177395.920768:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786177395.920771:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786177395.920774:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.920777:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177395.920779:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.920780:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177395.920784:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786177395.920787:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177395.920789:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177395.920792:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177395.920796:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177395.920799:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.920802:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177395.920804:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177395.920818:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786177395.920822:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786177395.920827:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177395.920829:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.920834:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177395.920838:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177395.920843:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177395.920853:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e2000. 00000100:00000001:0.0:1786177395.920861:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177395.920872:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177395.920883:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177395.920889:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177395.920894:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177395.920923:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177395.920927:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177395.920930:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786177395.920932:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177395.920933:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177395.920937:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177395.920939:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12070 00000400:00000200:0.0:1786177395.920943:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177395.920945:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177395.920947:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177395.920950:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.920952:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177395.920954:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.920956:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177395.920959:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177395.920960:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786177395.920962:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177395.920964:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786177395.920966:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786177395.920968:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786177395.920972:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.920974:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177395.920976:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.920978:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177395.920980:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786177395.920983:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177395.920985:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177395.920987:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177395.920994:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177395.920996:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.920998:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177395.921006:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786177395.921010:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.921013:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177395.921017:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786177395.921019:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786177395.921021:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177395.921023:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.921025:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177395.921026:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177395.921028:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177395.921031:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1800. 00000100:00000001:0.0:1786177395.921034:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177395.921039:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177395.921047:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177395.921049:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177395.921052:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177395.921061:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177395.921064:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177395.921067:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786177395.921069:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177395.921070:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177395.921073:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177395.921076:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12070 00000400:00000200:0.0:1786177395.921079:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177395.921082:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177395.921084:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177395.921086:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.921088:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177395.921090:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.921092:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177395.921095:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177395.921096:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786177395.921098:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177395.921100:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786177395.921102:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786177395.921104:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786177395.921107:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.921110:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177395.921112:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.921115:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177395.921118:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00000001:2.0:1786177395.921120:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1786177395.921121:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177395.921124:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177395.921127:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177395.921130:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177395.921132:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.921134:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177395.921136:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1786177395.921137:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000008:00000001:0.0:1786177395.921140:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.921142:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.921143:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:0.0:1786177395.921146:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786177395.921150:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786177395.921153:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177395.921155:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177395.921156:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177395.921157:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177395.921160:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177395.921162:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177395.921168:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3800. 00000100:00000001:0.0:1786177395.921173:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.921173:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177395.921178:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177395.921181:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1786177395.921185:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177395.921190:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177395.921193:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1786177395.921195:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.921196:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177395.921202:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235612160:192.168.202.132@tcp:250:kworker.0 00000100:00000040:0.0:1786177395.921204:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.921213:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1786177395.921219:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177395.921220:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1786177395.921222:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1786177395.921226:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177395.921227:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786177395.921230:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1786177395.921231:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1786177395.921232:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177395.921238:0:40394:0:(import.c:485:import_select_connection()) Process entered 02000000:00000010:2.0:1786177395.921240:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 00000100:00080000:0.0:1786177395.921241:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12070 02000000:00000001:2.0:1786177395.921246:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177395.921247:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177395.921252:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177395.921255:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177395.921258:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:0.0:1786177395.921260:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.921264:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177395.921266:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000400:00000010:2.0:1786177395.921266:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000100:00000001:0.0:1786177395.921272:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177395.921275:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177395.921277:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786177395.921279:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:2.0:1786177395.921279:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000020:00000040:0.0:1786177395.921282:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786177395.921286:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786177395.921288:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786177395.921291:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000200:2.0:1786177395.921292:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235612160, portal 25 00000100:00000001:0.0:1786177395.921294:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177395.921297:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177395.921298:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177395.921300:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000100:00000001:2.0:1786177395.921301:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794203392 : -110371915348224 : ffff9b9e09314700) 00000020:00000040:0.0:1786177395.921304:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786177395.921309:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000040:2.0:1786177395.921312:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177395.921313:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177395.921317:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177395.921320:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177395.921323:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.921325:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177395.921328:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1786177395.921337:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177395.921341:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000008:00000001:0.0:1786177395.921343:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177395.921345:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000001:0.0:1786177395.921347:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177395.921352:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235612160, offset 0 00000100:00000010:0.0:1786177395.921353:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000400:00000200:2.0:1786177395.921389:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000020:00000040:0.0:1786177395.921400:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 00000400:00000200:2.0:1786177395.921401:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:0.0:1786177395.921405:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177395.921408:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.921410:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177395.921421:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1786177395.921422:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 02000000:00000001:0.0:1786177395.921424:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177395.921430:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000400:00000200:2.0:1786177395.921431:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1786177395.921435:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177395.921439:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000100:00100000:0.0:1786177395.921443:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786177395.921447:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177395.921453:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177395.921462:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426849:426849:256:4294967295] 192.168.202.132@tcp LPNI seq info [426811:426811:8:4294967295] 00000100:00000001:0.0:1786177395.921468:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177395.921471:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177395.921471:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:0.0:1786177395.921478:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1786177395.921492:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00000001:0.0:1786177395.921493:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177395.921498:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1786177395.921503:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00080000:0.0:1786177395.921504:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786177395.921508:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177395.921509:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177395.921512:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000800:00000010:2.0:1786177395.921512:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000100:00080000:0.0:1786177395.921517:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12070 00000400:00000200:0.0:1786177395.921521:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000800:00000200:2.0:1786177395.921525:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000400:00000200:0.0:1786177395.921528:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177395.921531:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000800:00000200:2.0:1786177395.921533:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000100:00000040:0.0:1786177395.921536:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.921541:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1786177395.921544:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:0.0:1786177395.921546:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.921550:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000800:00000200:2.1:1786177395.921553:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177395.921560:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000020:00000001:0.0:1786177395.921563:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000800:00000010:2.0:1786177395.921564:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000020:00000040:0.0:1786177395.921566:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786177395.921568:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177395.921571:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786177395.921574:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000200:2.0:1786177395.921574:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:0.0:1786177395.921579:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786177395.921585:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177395.921586:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1786177395.921588:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177395.921593:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177395.921595:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177395.921595:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000020:00000040:0.0:1786177395.921600:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000400:00000010:2.0:1786177395.921600:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00080000:0.0:1786177395.921606:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177395.921608:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.921608:0:2392:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1786177395.921612:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177395.921615:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1786177395.921616:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786177395.921619:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.921622:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177395.921625:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786177395.921630:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.921633:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177395.921638:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000100:00000001:2.0:1786177395.921639:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:0.0:1786177395.921640:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177395.921645:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:2.0:1786177395.921645:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1786177395.921648:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177395.921652:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177395.921654:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177395.921657:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.921662:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177395.921664:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:2.0:1786177395.921665:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177395.921669:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.921674:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177395.921676:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1786177395.921683:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177395.921688:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177395.921689:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306746629 00000100:00000001:0.0:1786177395.921692:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177395.921697:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c1e0. 00000100:00000040:0.0:1786177395.921698:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.921708:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177395.921709:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177395.921716:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12080) 00000100:00000001:2.0:1786177395.921721:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306746629 00000100:00000010:2.0:1786177395.921726:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290ca80. 00000100:00000001:2.0:1786177395.921730:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177395.921736:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177395.921739:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.921742:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177395.921748:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177395/real 1786177395] req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177395.921760:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177395.921762:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177395.921764:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177395.921767:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177395.921769:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177395.921771:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177395.921775:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177395.921784:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.921791:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177395.921794:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.921797:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177395.921799:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177395.921804:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.921811:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177395.921816:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.921824:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177395.921828:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.921829:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177395.921831:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.921833:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177395.921836:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177395.921839:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177395.921843:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177395.921846:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177395.921851:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177395.921860:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235612160:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177395.921865:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177395.921868:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.921875:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177395.921876:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177395.921879:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177395.921881:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177395.921883:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177395.921885:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612288:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177395.921889:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177395.921891:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177395.921892:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177395.921894:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177395.921897:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177395.921899:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177395.921903:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177395.921905:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177395.921909:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177395.921912:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235612288, portal 10 00000100:00000001:2.0:1786177395.921914:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177395.921915:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794213248 : -110371915338368 : ffff9b9e09316d80) 00000100:00000040:2.0:1786177395.921922:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.921929:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177395.921932:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177395.921934:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177395.921937:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235612288, offset 0 00000400:00000200:2.0:1786177395.921941:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177395.921944:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177395.921948:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177395.921951:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177395.921954:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177395.921956:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177395.921959:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177395.921961:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426850:426850:256:4294967295] 192.168.202.132@tcp LPNI seq info [426812:426812:8:4294967295] 00000400:00000200:2.0:1786177395.921966:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177395.921972:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177395.921977:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177395.921980:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177395.921984:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177395.921987:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177395.921991:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177395.921993:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177395.921997:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177395.921999:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177395.922002:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177395.922005:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177395.922008:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177395.922010:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177395.922013:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177395.922016:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922024:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177395.922027:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922034:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.922036:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177395.922038:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177395.922043:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177395.922046:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177395.922048:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306742792 00000100:00000001:2.0:1786177395.922051:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306746629 00000100:00000001:2.0:1786177395.922053:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306742792 00000100:00000001:2.0:1786177395.922056:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306746629 00000100:00000001:2.0:1786177395.922058:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177395.922060:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.922062:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177395.922065:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177395/real 1786177395] req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177395.922073:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177395.922075:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177395.922077:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177395.922079:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177395.922081:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177395.922083:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177395.922086:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177395.922095:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922102:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177395.922104:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922105:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177395.922106:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177395.922110:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922117:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177395.922121:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922128:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177395.922131:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922133:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177395.922134:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922136:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177395.922138:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177395.922141:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177395.922145:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177395.922147:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177395.922151:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177395.922159:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612288:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177395.922165:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922167:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177395.922170:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314700 x1872930235612160/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922178:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177395.922182:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177395.922185:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177395.922188:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177395.922190:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177395.922192:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177395.922195:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177395.922198:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:2.0:1786177395.922203:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177395.922205:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177395.922207:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314700. 00000100:00000001:2.0:1786177395.922211:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177395.922212:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922214:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177395.922217:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316d80 x1872930235612288/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922224:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177395.922226:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177395.922227:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177395.922229:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177395.922230:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177395.922232:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177395.922234:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177395.922235:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:2.0:1786177395.922237:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177395.922238:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177395.922239:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316d80. 00000100:00000001:2.0:1786177395.922242:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177395.922243:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922246:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922256:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177395.922259:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.922261:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177395.922262:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177395.922264:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177395.922267:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922274:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177395.922276:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177395.922278:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177395.922279:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177395.922281:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177395.922284:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612416:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177395.922288:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177395.922290:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177395.922291:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177395.922293:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177395.922296:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177395.922298:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177395.922301:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177395.922303:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177395.922306:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177395.922309:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235612416, portal 10 00000100:00000001:2.0:1786177395.922310:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177395.922312:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215040 : -110371915336576 : ffff9b9e09317480) 00000100:00000040:2.0:1786177395.922319:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922325:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177395.922327:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177395.922330:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177395.922333:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235612416, offset 0 00000400:00000200:2.0:1786177395.922337:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177395.922340:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177395.922344:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177395.922348:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177395.922350:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177395.922352:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177395.922354:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177395.922377:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426851:426851:256:4294967295] 192.168.202.132@tcp LPNI seq info [426813:426813:8:4294967295] 00000400:00000200:2.0:1786177395.922382:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177395.922388:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177395.922393:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177395.922396:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177395.922400:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177395.922403:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177395.922407:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177395.922409:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177395.922423:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177395.922425:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177395.922428:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177395.922432:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177395.922434:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177395.922436:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177395.922438:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177395.922441:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922449:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177395.922452:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922459:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.922461:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177395.922464:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177395.922467:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177395.922470:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177395.922472:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306742793 00000100:00000001:2.0:1786177395.922474:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306746629 00000100:00000001:2.0:1786177395.922477:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306742793 00000100:00000001:2.0:1786177395.922480:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306746629 00000100:00000001:2.0:1786177395.922482:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177395.922484:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.922486:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177395.922490:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177395/real 1786177395] req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177395.922498:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177395.922499:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177395.922501:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177395.922504:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177395.922506:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177395.922508:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177395.922512:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177395.922520:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922527:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177395.922528:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922530:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177395.922532:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177395.922536:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922544:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177395.922548:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922555:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177395.922558:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922560:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177395.922562:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922563:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177395.922571:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177395.922574:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177395.922578:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177395.922580:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177395.922584:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177395.922592:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612416:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177395.922596:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177395.922600:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922607:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177395.922608:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177395.922610:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177395.922612:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177395.922614:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177395.922617:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09315500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612544:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177395.922620:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177395.922622:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177395.922624:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177395.922625:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177395.922628:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177395.922630:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177395.922634:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177395.922636:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177395.922639:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177395.922642:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235612544, portal 4 00000100:00000001:2.0:1786177395.922644:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177395.922646:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794206976 : -110371915344640 : ffff9b9e09315500) 00000100:00000040:2.0:1786177395.922653:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922660:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177395.922663:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177395.922665:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177395.922667:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235612544, offset 0 00000400:00000200:2.0:1786177395.922671:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177395.922675:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177395.922678:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177395.922681:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177395.922684:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177395.922686:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177395.922689:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177395.922692:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426852:426852:256:4294967295] 192.168.202.132@tcp LPNI seq info [426814:426814:8:4294967295] 00000400:00000200:2.0:1786177395.922696:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177395.922702:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177395.922707:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177395.922710:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177395.922714:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177395.922717:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177395.922721:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177395.922724:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177395.922727:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177395.922729:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177395.922732:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177395.922735:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177395.922738:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177395.922740:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177395.922742:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177395.922745:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922753:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177395.922757:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922763:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.922765:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177395.922768:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177395.922770:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177395.922773:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177395.922775:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306742793 00000100:00000001:2.0:1786177395.922778:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306746629 00000100:00000001:2.0:1786177395.922781:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306742793 00000100:00000001:2.0:1786177395.922783:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306746629 00000100:00000001:2.0:1786177395.922785:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177395.922787:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.922789:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177395.922792:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177395/real 1786177395] req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177395.922800:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177395.922802:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177395.922803:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177395.922806:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177395.922808:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177395.922809:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177395.922813:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177395.922821:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922828:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177395.922830:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922831:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177395.922833:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177395.922837:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922844:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177395.922848:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922855:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177395.922857:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922859:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177395.922860:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.922862:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177395.922864:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177395.922866:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177395.922870:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177395.922872:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177395.922875:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177395.922884:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09315500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612544:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177395.922888:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177395.922891:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922899:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177395.922900:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177395.922902:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177395.922904:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177395.922906:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177395.922909:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612672:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177395.922912:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177395.922915:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177395.922916:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177395.922918:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177395.922921:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177395.922923:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177395.922926:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177395.922929:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177395.922931:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177395.922934:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235612672, portal 4 00000100:00000001:2.0:1786177395.922936:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177395.922937:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794204288 : -110371915347328 : ffff9b9e09314a80) 00000100:00000040:2.0:1786177395.922942:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.922949:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177395.922951:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177395.922954:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177395.922956:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235612672, offset 0 00000400:00000200:2.0:1786177395.922960:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177395.922963:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177395.922966:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177395.922969:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177395.922971:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177395.922974:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177395.922976:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177395.922979:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426853:426853:256:4294967295] 192.168.202.132@tcp LPNI seq info [426815:426815:8:4294967295] 00000400:00000200:2.0:1786177395.922983:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177395.922989:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177395.922994:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177395.922997:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177395.923000:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177395.923003:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1786177395.923007:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177395.923009:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 37 secs later) 00000800:00000100:2.0:1786177395.923013:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177395.923016:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177395.923019:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177395.923022:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177395.923025:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177395.923027:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177395.923029:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177395.923032:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.923040:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177395.923043:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177420 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.923050:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.923051:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177395.923054:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177395.923057:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177395.923060:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177395.923062:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306742795 00000100:00000001:2.0:1786177395.923064:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306746629 00000100:00000001:2.0:1786177395.923066:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306742795 00000100:00000001:2.0:1786177395.923068:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306746629 00000100:00000001:2.0:1786177395.923071:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177395.923073:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.923075:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177395.923078:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177395/real 1786177395] req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177395.923086:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177395.923088:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177395.923090:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177395.923092:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177395.923094:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177395.923096:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177395.923100:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177395.923108:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.923115:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177395.923116:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.923118:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177395.923120:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177395.923123:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.923130:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177395.923134:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.923141:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177395.923144:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.923145:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177395.923147:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.923148:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177395.923149:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177395.923152:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177395.923156:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177395.923158:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177395.923161:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177395.923170:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612672:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177395.923173:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.923175:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177395.923179:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317480 x1872930235612416/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.923186:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177395.923188:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177395.923190:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177395.923192:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177395.923194:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177395.923195:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177395.923197:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177395.923199:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:2.0:1786177395.923202:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177395.923204:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177395.923205:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317480. 00000100:00000001:2.0:1786177395.923208:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177395.923209:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.923211:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177395.923214:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09315500 x1872930235612544/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.923221:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177395.923222:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177395.923224:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177395.923226:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177395.923227:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177395.923228:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177395.923230:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177395.923232:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:2.0:1786177395.923234:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177395.923235:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177395.923236:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09315500. 00000100:00000001:2.0:1786177395.923239:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177395.923240:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.923242:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177395.923245:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314a80 x1872930235612672/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177420 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177395.923252:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177395.923254:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177395.923256:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177395.923258:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177395.923259:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177395.923261:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177395.923262:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177395.923264:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:2.0:1786177395.923266:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177395.923267:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177395.923269:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314a80. 00000100:00000001:2.0:1786177395.923271:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177395.923272:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.923274:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177395.923278:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177395.923279:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177395.923282:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177395.923284:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177396.620578:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177396.620585:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177396.620586:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177396.620591:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177396.620594:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177396.620596:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177396.620598:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177396.620600:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177396.620603:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177396.812491:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177396.812499:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177396.812502:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177396.812508:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177396.812513:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1786177396.812517:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1786177396.812518:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1786177396.812526:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177396.812527:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177396.812531:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177396.812536:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177396.812537:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177396.812540:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177396.812555:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177396.812559:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177396.812692:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177396.812697:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177396.812701:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177397.644666:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177397.644678:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177397.644681:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177397.644689:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177397.644694:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177397.644699:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177397.644702:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177397.644707:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177397.644710:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177397.836440:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177397.836448:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177397.836454:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177397.836455:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177397.836459:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177397.836460:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177397.836587:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177397.836591:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177397.836594:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177397.837671:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177397.837679:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177397.837682:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177397.837688:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177397.837692:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177397.837697:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177397.837700:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177397.837704:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177397.837708:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177398.668612:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177398.668620:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177398.668622:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177398.668628:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177398.668631:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177398.668635:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177398.668638:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177398.668640:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177398.668650:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177398.863623:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177398.863631:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177398.863634:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177398.863639:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177398.863642:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177398.863646:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177398.863649:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177398.863652:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177398.863655:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177398.864036:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177398.864039:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177398.864043:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177398.864045:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177398.864048:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177398.864049:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177398.864134:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177398.864138:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177398.864141:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177399.692971:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177399.692981:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177399.692983:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177399.692990:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177399.692994:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177399.692998:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177399.693001:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177399.693004:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177399.693007:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177399.820481:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306746629, at 4306746688 00000100:00000010:0.0:1786177399.820496:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c1e0. 00000100:00000040:1.0:1786177399.820537:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306746629, at 4306746688 00000100:00000010:1.0:1786177399.820544:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290ca80. 00000100:00000001:1.0:1786177399.884492:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177399.884500:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177399.884503:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177399.884508:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177399.884512:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177399.884522:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177399.884525:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177399.884528:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177399.884531:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177399.884713:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177399.884716:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177399.884720:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177399.884721:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177399.884724:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177399.884725:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1786177399.887004:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1786177399.887010:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1786177399.887014:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177400.716605:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177400.716613:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177400.716615:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177400.716621:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177400.716625:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177400.716629:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177400.716632:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177400.716635:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177400.716638:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177400.908943:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177400.908950:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177400.908955:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177400.908956:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177400.908959:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177400.908960:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177400.909057:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177400.909073:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177400.909077:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177400.910277:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177400.910284:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177400.910286:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177400.910291:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177400.910295:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177400.910299:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177400.910301:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177400.910304:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177400.910307:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177401.041572:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177401.041615:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1786177401.041617:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177401.041620:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177401.041624:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177401.041630:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12075 00000400:00000200:0.0:1786177401.041645:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177401.041650:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177401.041653:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177401.041656:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.041659:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177401.041662:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.041664:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177401.041668:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177401.041670:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:0.0:1786177401.041673:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177401.041675:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:0.0:1786177401.041678:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:0.0:1786177401.041680:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:0.0:1786177401.041683:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.041685:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177401.041687:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.041689:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177401.041692:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:0.0:1786177401.041695:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177401.041698:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177401.041701:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177401.041705:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177401.041708:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.041710:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1786177401.041712:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177401.041722:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314a80. 00000020:00000040:0.0:1786177401.041726:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:0.0:1786177401.041730:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177401.041731:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.041734:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177401.041737:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177401.041740:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177401.041750:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3000. 00000100:00000001:0.0:1786177401.041756:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177401.041764:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177401.041771:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177401.041774:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177401.041777:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051bc2fe4] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.041853:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177401.041857:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177401.041860:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177401.041866:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.041882:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177401.041884:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177401.041888:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177401.041890:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177401.041892:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177401.041896:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235612800:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177401.041900:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177401.041905:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177401.041908:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177401.041910:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177401.041915:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177401.041917:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177401.041921:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177401.041924:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177401.041929:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177401.041935:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235612800, portal 25 00000100:00000001:2.0:1786177401.041937:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177401.041938:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794204288 : -110371915347328 : ffff9b9e09314a80) 00000100:00000040:2.0:1786177401.041942:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.041947:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177401.041949:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177401.041951:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177401.041953:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235612800, offset 0 00000400:00000200:2.0:1786177401.041958:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177401.041964:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177401.041969:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177401.041973:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177401.041976:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177401.041979:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177401.041981:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177401.041992:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426854:426854:256:4294967295] 192.168.202.132@tcp LPNI seq info [426816:426816:8:4294967295] 00000400:00000200:2.0:1786177401.041996:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177401.042003:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177401.042007:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177401.042012:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177401.042020:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177401.042023:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786177401.042028:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177401.042032:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786177401.042037:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177401.042039:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177401.042046:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177401.042050:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177401.042055:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177401.042058:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177401.042062:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177401.042066:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177401.042093:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177401.042095:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177401.042098:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1786177401.042099:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177401.042100:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177401.042103:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177401.042105:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12075 00000400:00000200:0.0:1786177401.042109:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177401.042112:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177401.042114:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177401.042116:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.042117:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177401.042118:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.042120:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177401.042121:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177401.042122:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:0.0:1786177401.042123:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177401.042124:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:0.0:1786177401.042125:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:0.0:1786177401.042127:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:0.0:1786177401.042129:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.042130:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177401.042131:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.042134:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177401.042136:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:0.0:1786177401.042139:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177401.042141:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177401.042143:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177401.042147:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177401.042148:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.042149:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177401.042153:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786177401.042155:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.042157:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177401.042161:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09315500. 00000020:00000040:0.0:1786177401.042163:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786177401.042165:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177401.042167:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.042168:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177401.042170:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177401.042172:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177401.042175:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0c00. 00000100:00000001:0.0:1786177401.042178:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177401.042182:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177401.042187:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177401.042189:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177401.042193:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a4c5f4b1] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.057704:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177401.057732:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.057745:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.057747:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177401.057763:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177401.057770:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177401.057772:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306751759 00000100:00000010:2.0:1786177401.057781:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290cea0. 00000100:00000001:2.0:1786177401.057787:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177401.057793:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306751759 00000100:00000010:2.0:1786177401.057795:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c240. 00000100:00000001:2.0:1786177401.057797:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177401.057799:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177401.057802:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.057805:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177401.057808:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177401/real 1786177401] req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177401.057820:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177401.057822:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177401.057825:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177401.057839:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177401.057842:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177401.057846:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177401.057851:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177401.057861:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.057869:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177401.057870:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.057872:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177401.057880:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177401.057883:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.057887:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177401.057891:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.057895:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177401.057897:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.057898:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177401.057899:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.057901:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177401.057903:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177401.057905:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177401.057908:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177401.057909:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177401.057912:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177401.057917:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235612800:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177401.057921:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.057923:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177401.057926:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314a80 x1872930235612800/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.057938:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177401.057941:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177401.057944:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177401.057948:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177401.057950:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177401.057951:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177401.057952:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177401.057954:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:2.0:1786177401.057957:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177401.057959:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177401.057960:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314a80. 00000100:00000001:2.0:1786177401.057963:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177401.057965:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.057967:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.057974:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177401.057976:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.057977:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177401.057979:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177401.057980:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177401.057992:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.057999:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177401.058002:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177401.058005:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177401.058009:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177401.058011:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177401.058014:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09315500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612928:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177401.058018:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177401.058022:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177401.058024:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177401.058027:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177401.058032:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177401.058034:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177401.058039:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177401.058043:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177401.058047:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177401.058052:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235612928, portal 10 00000100:00000001:2.0:1786177401.058055:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177401.058056:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794206976 : -110371915344640 : ffff9b9e09315500) 00000100:00000040:2.0:1786177401.058064:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.058071:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177401.058073:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177401.058076:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177401.058079:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235612928, offset 0 00000400:00000200:2.0:1786177401.058085:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177401.058092:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177401.058100:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177401.058103:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177401.058107:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177401.058110:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177401.058112:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177401.058117:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426855:426855:256:4294967295] 192.168.202.132@tcp LPNI seq info [426817:426817:8:4294967295] 00000400:00000200:2.0:1786177401.058121:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177401.058130:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177401.058135:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177401.058139:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177401.058144:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177401.058146:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786177401.058150:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177401.058153:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786177401.058155:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177401.058157:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177401.058160:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177401.058163:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177401.058166:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177401.058168:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177401.058170:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177401.058174:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.058182:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177401.058185:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.058189:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.058190:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177401.058193:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177401.058232:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177401.058235:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177401.058236:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306747922 00000100:00000001:2.0:1786177401.058238:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306751759 00000100:00000001:2.0:1786177401.058240:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306747922 00000100:00000001:2.0:1786177401.058241:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306751759 00000100:00000001:2.0:1786177401.058243:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177401.058245:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.058247:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177401.058251:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177401/real 1786177401] req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177401.058259:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177401.058260:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177401.058262:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177401.058264:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177401.058267:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177401.058269:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177401.058272:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177401.058279:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.058290:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177401.058292:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.058293:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177401.058294:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177401.058298:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.058305:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177401.058309:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.058315:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177401.058316:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.058317:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177401.058318:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.058319:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177401.058321:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177401.058322:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177401.058324:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177401.058326:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177401.058328:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177401.058334:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09315500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235612928:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177401.058337:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.058339:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177401.058342:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09315500 x1872930235612928/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.058347:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177401.058349:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177401.058351:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177401.058353:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177401.058354:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177401.058356:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177401.058389:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177401.058391:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:2.0:1786177401.058395:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177401.058396:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177401.058397:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09315500. 00000100:00000001:2.0:1786177401.058400:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177401.058401:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.058403:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.058406:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177401.058415:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.058416:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177401.058417:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.070780:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177401.070791:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177401.070795:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1786177401.070797:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177401.070800:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177401.070805:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177401.070813:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12075 00000400:00000200:0.0:1786177401.070821:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177401.070833:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177401.070836:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177401.070839:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.070842:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177401.070845:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.070847:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177401.070851:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177401.070853:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:0.0:1786177401.070855:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177401.070858:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:0.0:1786177401.070861:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:0.0:1786177401.070863:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:0.0:1786177401.070865:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.070867:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177401.070869:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.070879:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177401.070882:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:0.0:1786177401.070885:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177401.070888:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177401.070891:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177401.070899:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177401.070902:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.070904:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177401.070908:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1786177401.070911:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.070913:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177401.070923:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09317480. 00000020:00000040:0.0:1786177401.070927:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:0.0:1786177401.070931:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177401.070932:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.070935:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177401.070938:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177401.070942:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177401.070949:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e3400. 00000100:00000001:0.0:1786177401.070956:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177401.070967:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177401.070975:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177401.070978:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177401.070981:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043993004] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177401.070996:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177401.070998:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177401.071002:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1786177401.071003:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177401.071005:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177401.071008:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177401.071011:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12075 00000400:00000200:0.0:1786177401.071015:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177401.071018:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177401.071020:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177401.071022:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.071024:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177401.071026:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.071028:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177401.071030:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177401.071031:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:0.0:1786177401.071033:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177401.071036:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:0.0:1786177401.071038:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:0.0:1786177401.071040:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:0.0:1786177401.071043:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.071045:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177401.071047:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.071049:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177401.071051:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:0.0:1786177401.071053:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177401.071055:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177401.071056:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177401.071059:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177401.071060:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.071061:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177401.071063:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786177401.071065:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.071066:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177401.071069:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09316d80. 00000020:00000040:0.0:1786177401.071070:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177401.071071:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177401.071072:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.071074:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177401.071075:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177401.071076:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177401.071079:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e1c00. 00000100:00000001:0.0:1786177401.071081:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177401.071084:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177401.071089:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177401.071091:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177401.071093:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9167148] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177401.071098:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1786177401.071100:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1786177401.071102:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1786177401.071103:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1786177401.071104:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1786177401.071105:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1786177401.071107:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12075 00000400:00000200:0.0:1786177401.071109:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177401.071111:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:0.0:1786177401.071112:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1786177401.071113:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.071114:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177401.071116:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.071117:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:0.0:1786177401.071118:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:0.0:1786177401.071119:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:0.0:1786177401.071120:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1786177401.071121:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:0.0:1786177401.071122:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:0.0:1786177401.071123:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:0.0:1786177401.071125:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.071126:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1786177401.071127:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:0.0:1786177401.071128:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:0.0:1786177401.071129:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:0.0:1786177401.071130:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:0.0:1786177401.071132:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177401.071133:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1786177401.071134:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177401.071136:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.071137:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1786177401.071138:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1786177401.071139:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.071140:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1786177401.071142:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e09314700. 00000020:00000040:0.0:1786177401.071143:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:0.0:1786177401.071144:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1786177401.071145:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177401.071146:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1786177401.071146:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1786177401.071147:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1786177401.071149:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e319e0000. 00000100:00000001:0.0:1786177401.071150:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177401.071153:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1786177401.071157:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1786177401.071158:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177401.071160:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003fe53e8c] to pc [ptlrpcd_rcv+-1] req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177401.071164:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177401.071168:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12085) 00000100:00000001:2.0:1786177401.072540:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177401.072546:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177401.072548:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177401.072556:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.072567:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177401.072569:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177401.072573:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177401.072577:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177401.072579:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177401.072581:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613056:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177401.072585:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177401.072589:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177401.072592:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177401.072595:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177401.072602:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177401.072604:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177401.072610:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177401.072613:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177401.072619:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177401.072626:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235613056, portal 10 00000100:00000001:2.0:1786177401.072628:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177401.072629:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794215040 : -110371915336576 : ffff9b9e09317480) 00000100:00000040:2.0:1786177401.072636:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.072641:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177401.072643:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177401.072645:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177401.072648:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235613056, offset 0 00000400:00000200:2.0:1786177401.072653:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177401.072661:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177401.072668:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177401.072673:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177401.072677:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177401.072681:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177401.072684:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177401.072688:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426856:426856:256:4294967295] 192.168.202.132@tcp LPNI seq info [426818:426818:8:4294967295] 00000400:00000200:2.0:1786177401.072692:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177401.072702:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177401.072717:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177401.072723:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177401.072730:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177401.072734:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786177401.072740:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177401.072744:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786177401.072748:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177401.072751:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177401.072756:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177401.072762:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177401.072766:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177401.072769:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177401.072774:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177401.072778:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.072788:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177401.072791:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.072798:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.072800:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177401.072806:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177401.072845:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177401.072849:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177401.072853:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306747943 00000100:00000001:2.0:1786177401.072856:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306751759 00000100:00000001:2.0:1786177401.072858:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306747943 00000100:00000001:2.0:1786177401.072861:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306751759 00000100:00000001:2.0:1786177401.072863:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177401.072866:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.072868:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177401.073112:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177401/real 1786177401] req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177401.073124:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177401.073126:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177401.073129:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177401.073132:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177401.073134:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177401.073136:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177401.073140:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177401.073148:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.073155:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177401.073157:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.073160:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177401.073162:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177401.073166:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.073173:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177401.073177:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.073185:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177401.073188:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.073189:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177401.073191:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.073193:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177401.073196:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177401.073199:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177401.073203:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177401.073205:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177401.073209:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177401.073218:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613056:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177401.073222:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177401.073225:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.073232:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177401.073234:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177401.073236:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177401.073238:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177401.073240:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177401.073244:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613184:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177401.073247:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177401.073250:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177401.073252:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177401.073253:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177401.073256:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177401.073258:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177401.073262:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177401.073264:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177401.073267:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177401.073270:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235613184, portal 4 00000100:00000001:2.0:1786177401.073272:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177401.073273:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794213248 : -110371915338368 : ffff9b9e09316d80) 00000100:00000040:2.0:1786177401.073280:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.073287:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177401.073289:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177401.073292:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177401.073294:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235613184, offset 0 00000400:00000200:2.0:1786177401.073298:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177401.073302:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177401.073306:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177401.073309:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177401.073312:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177401.073314:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177401.073316:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177401.073319:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426857:426857:256:4294967295] 192.168.202.132@tcp LPNI seq info [426819:426819:8:4294967295] 00000400:00000200:2.0:1786177401.073323:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177401.073330:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177401.073334:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177401.073338:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177401.073342:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177401.073345:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786177401.073350:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177401.073352:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786177401.073356:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177401.073714:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177401.073718:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177401.073722:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177401.073726:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177401.073728:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177401.073731:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177401.073735:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.073743:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177401.073746:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.073752:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.073753:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177401.073757:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177401.073761:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177401.073765:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177401.073767:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306747944 00000100:00000001:2.0:1786177401.073770:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306751759 00000100:00000001:2.0:1786177401.073772:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306747944 00000100:00000001:2.0:1786177401.073774:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306751759 00000100:00000001:2.0:1786177401.073776:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177401.073778:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.073780:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177401.073784:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177401/real 1786177401] req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177401.073791:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177401.073793:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177401.073795:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177401.073797:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177401.073799:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177401.073801:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177401.073804:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177401.073812:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.073818:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177401.073820:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.073821:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177401.073823:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177401.074111:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074117:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177401.074120:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074125:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177401.074126:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.074128:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177401.074129:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.074130:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177401.074131:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177401.074133:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177401.074135:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177401.074136:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177401.074138:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177401.074147:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09316d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613184:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177401.074150:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177401.074153:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074157:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177401.074158:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177401.074160:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177401.074161:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177401.074162:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177401.074164:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613312:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177401.074166:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177401.074168:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177401.074170:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177401.074171:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177401.074174:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177401.074176:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177401.074178:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177401.074180:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177401.074183:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177401.074186:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235613312, portal 4 00000100:00000001:2.0:1786177401.074187:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177401.074188:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633701794203392 : -110371915348224 : ffff9b9e09314700) 00000100:00000040:2.0:1786177401.074192:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074196:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177401.074197:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177401.074198:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177401.074200:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235613312, offset 0 00000400:00000200:2.0:1786177401.074203:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177401.074207:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177401.074211:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177401.074213:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177401.074215:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177401.074217:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177401.074219:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177401.074221:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426858:426858:256:4294967295] 192.168.202.132@tcp LPNI seq info [426820:426820:8:4294967295] 00000400:00000200:2.0:1786177401.074224:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177401.074228:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177401.074231:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177401.074233:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177401.074237:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177401.074239:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1786177401.074244:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177401.074247:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1786177401.074250:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177401.074252:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177401.074255:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177401.074258:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177401.074261:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177401.074262:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177401.074265:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177401.074268:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074275:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177401.074278:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177426 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074284:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.074286:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177401.074289:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177401.074293:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177401.074296:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177401.074298:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306747946 00000100:00000001:2.0:1786177401.074301:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306751759 00000100:00000001:2.0:1786177401.074302:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306747946 00000100:00000001:2.0:1786177401.074304:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306751759 00000100:00000001:2.0:1786177401.074305:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177401.074306:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.074307:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177401.074310:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177401/real 1786177401] req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177401.074315:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177401.074316:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177401.074317:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177401.074318:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177401.074320:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177401.074321:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177401.074323:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177401.074328:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074333:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177401.074334:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.074335:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177401.074336:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177401.074339:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074345:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177401.074349:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074355:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177401.074394:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.074395:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177401.074396:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.074397:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177401.074398:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177401.074400:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177401.074403:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177401.074404:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177401.074407:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177401.074422:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e09314700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613312:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177401.074427:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.074429:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177401.074433:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09317480 x1872930235613056/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074437:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177401.074439:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177401.074441:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177401.074443:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177401.074445:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177401.074446:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177401.074448:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177401.074450:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:2.0:1786177401.074453:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177401.074454:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177401.074455:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09317480. 00000100:00000001:2.0:1786177401.074458:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177401.074459:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.074459:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177401.074461:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09316d80 x1872930235613184/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074469:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177401.074470:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177401.074471:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177401.074472:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177401.074473:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177401.074474:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177401.074474:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177401.074475:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:2.0:1786177401.074476:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177401.074477:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177401.074478:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09316d80. 00000100:00000001:2.0:1786177401.074479:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177401.074480:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.074480:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177401.074482:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e09314700 x1872930235613312/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177426 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177401.074485:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177401.074486:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177401.074486:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaac00. 02000000:00000001:2.0:1786177401.074487:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177401.074488:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177401.074488:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177401.074489:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177401.074490:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:2.0:1786177401.074491:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177401.074492:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177401.074492:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e09314700. 00000100:00000001:2.0:1786177401.074494:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177401.074495:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.074496:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177401.074502:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177401.075614:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177401.075619:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177401.075621:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177401.740576:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177401.740584:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177401.740587:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177401.740593:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177401.740597:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177401.740600:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177401.740603:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177401.740605:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177401.740608:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177401.932681:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177401.932688:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177401.932694:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177401.932697:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177401.932699:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177401.932702:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177401.932705:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177401.932707:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177401.932709:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177401.932712:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177401.932716:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177401.932717:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177402.016640:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177402.016654:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177402.016659:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177402.016747:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177402.016751:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177402.016753:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177402.016757:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177402.016761:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177402.016764:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177402.016767:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177402.016769:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177402.016772:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177402.764506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177402.764514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177402.764517:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177402.764522:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177402.764525:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177402.764529:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177402.764532:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177402.764534:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177402.764538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177402.956483:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177402.956491:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177402.956497:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177402.956498:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177402.956502:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177402.956503:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177402.959576:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177402.959583:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177402.959587:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177403.020641:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177403.020648:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177403.020650:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177403.020655:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177403.020659:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177403.020663:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177403.020666:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177403.020668:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177403.020671:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177403.788488:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177403.788496:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177403.788498:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177403.788503:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177403.788508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177403.788512:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177403.788514:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177403.788517:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177403.788520:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177403.981192:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177403.981200:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177403.981202:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177403.981206:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177403.981210:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177403.981213:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177403.981215:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177403.981217:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177403.981222:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177403.981223:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177403.981227:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177403.981229:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177403.981230:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177403.981232:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177403.981233:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177403.981234:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177403.981236:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177403.981237:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177403.981406:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177403.981410:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177403.981413:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177404.044534:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177404.044542:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177404.044544:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177404.044550:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177404.044553:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177404.044557:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177404.044560:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177404.044562:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177404.044565:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177404.812598:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177404.812607:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177404.812609:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177404.812614:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177404.812618:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177404.812622:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177404.812624:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177404.812626:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177404.812630:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177404.941397:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306751759, at 4306751809 00000100:00000010:1.0:1786177404.941415:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c240. 00000100:00000040:0.0:1786177404.941986:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306751759, at 4306751809 00000100:00000010:0.0:1786177404.941995:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290cea0. 00010000:00000001:0.0:1786177405.004499:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177405.004508:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177405.004515:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177405.004516:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177405.004519:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177405.004520:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177405.004633:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177405.004637:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177405.004651:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177405.068771:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177405.068781:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177405.068784:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177405.068790:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177405.068794:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177405.068798:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177405.068800:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177405.068803:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177405.068806:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177405.837110:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177405.837138:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177405.837140:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177405.837145:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177405.837149:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177405.837153:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177405.837156:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177405.837158:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177405.837161:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177406.028619:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177406.028626:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177406.028632:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177406.028634:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177406.028637:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177406.028638:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177406.028831:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177406.028835:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177406.028838:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177406.092487:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177406.092495:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177406.092497:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177406.092502:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177406.092506:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177406.092509:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177406.092512:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177406.092514:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177406.092518:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177406.156520:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177406.156531:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1786177406.156533:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177406.156536:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177406.156540:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177406.156547:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12080 00000400:00000200:1.0:1786177406.156556:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177406.156562:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177406.156565:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177406.156568:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.156572:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177406.156575:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.156577:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177406.156580:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177406.156582:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:1.0:1786177406.156585:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177406.156588:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:1.0:1786177406.156591:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:1.0:1786177406.156637:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:1.0:1786177406.156640:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.156642:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177406.156644:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.156646:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177406.156648:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:1.0:1786177406.156651:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177406.156653:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177406.156656:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177406.156659:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177406.156662:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.156664:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1786177406.156666:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1786177406.156676:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b9180. 00000020:00000040:1.0:1786177406.156679:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:1.0:1786177406.156683:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177406.156685:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.156688:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177406.156690:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177406.156694:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177406.156701:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af800. 00000100:00000001:1.0:1786177406.156707:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177406.156717:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1786177406.156727:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1786177406.156730:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177406.156734:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001d01dc3b] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177406.156762:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177406.156766:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177406.156769:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1786177406.156771:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1786177406.156773:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177406.156776:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177406.156778:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12080 00000400:00000200:1.0:1786177406.156782:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177406.156785:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177406.156788:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1786177406.156790:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.156792:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177406.156794:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.156796:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177406.156798:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177406.156799:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:1.0:1786177406.156801:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177406.156803:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:1.0:1786177406.156804:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:1.0:1786177406.156806:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:1.0:1786177406.156809:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.156810:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177406.156812:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.156814:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177406.156816:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:1.0:1786177406.156819:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177406.156821:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177406.156824:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177406.156829:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177406.156832:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.156833:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177406.156836:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:1.0:1786177406.156838:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177406.156843:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.156843:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177406.156848:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.156848:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:1.0:1786177406.156853:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328b8a80. 00000100:00000040:2.0:1786177406.156856:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1786177406.156857:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:1.0:1786177406.156862:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177406.156864:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.156869:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177406.156871:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177406.156873:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1786177406.156877:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261ad800. 00000100:00000001:1.0:1786177406.156880:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177406.156887:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177406.156887:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177406.156890:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177406.156897:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1786177406.156900:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1786177406.156901:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1786177406.156905:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177406.156906:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177406.156912:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002bb61190] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177406.156913:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235613440:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177406.156921:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177406.156926:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1786177406.156929:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177406.156930:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1786177406.156936:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1786177406.156936:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1786177406.156941:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1786177406.156942:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000010:2.0:1786177406.156942:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 00000100:00080000:1.0:1786177406.156946:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1786177406.156946:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.156952:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177406.156956:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177406.156961:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177406.156966:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00080000:1.0:1786177406.156972:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12080 00000100:00000200:2.0:1786177406.156973:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235613440, portal 25 00000100:00000001:2.0:1786177406.156978:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1786177406.156979:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177406.156981:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487986560 : -110371221565056 : ffff9b9e328b9180) 00000400:00000200:1.0:1786177406.156984:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177406.156987:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177406.156989:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177406.156993:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.156997:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177406.157001:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177406.157006:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1786177406.157007:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177406.157010:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177406.157014:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000100:00000040:2.0:1786177406.157015:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000020:00000001:1.0:1786177406.157016:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177406.157020:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:1.0:1786177406.157022:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:1.0:1786177406.157024:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:1.0:1786177406.157027:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000010:2.0:1786177406.157028:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000001:1.0:1786177406.157030:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:2.0:1786177406.157033:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235613440, offset 0 00000100:00000040:1.0:1786177406.157034:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157039:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177406.157042:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000400:00000200:2.0:1786177406.157045:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:1.0:1786177406.157048:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157052:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177406.157054:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177406.157057:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1786177406.157057:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000001:1.0:1786177406.157062:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.157064:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:2.0:1786177406.157066:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000008:00100000:1.0:1786177406.157067:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1786177406.157072:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.157074:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177406.157074:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:1.0:1786177406.157080:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328baa00. 00000400:00000200:2.0:1786177406.157081:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000020:00000040:1.0:1786177406.157084:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000400:00000200:2.0:1786177406.157088:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1786177406.157090:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1786177406.157093:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177406.157094:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157097:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1786177406.157100:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1786177406.157101:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426859:426859:256:4294967295] 192.168.202.132@tcp LPNI seq info [426821:426821:8:4294967295] 02000000:00000001:1.0:1786177406.157104:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177406.157109:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:1.0:1786177406.157111:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261aec00. 00000100:00000001:1.0:1786177406.157116:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177406.157121:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000100:00100000:1.0:1786177406.157125:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.0:1786177406.157132:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157139:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:2.0:1786177406.157140:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000100:00000001:1.0:1786177406.157143:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177406.157151:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000040:1.0:1786177406.157152:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ebce398] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1786177406.157156:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00000001:1.0:1786177406.157167:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786177406.157167:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177406.157172:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000100:00080000:1.0:1786177406.157173:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1786177406.157180:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1786177406.157180:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157184:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000010:2.0:1786177406.157185:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000100:00080000:1.0:1786177406.157187:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1786177406.157191:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1786177406.157192:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1786177406.157198:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12080 00000400:00000200:2.0:1786177406.157201:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1786177406.157205:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177406.157210:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177406.157212:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177406.157218:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177406.157221:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177406.157225:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177406.157229:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.157238:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177406.157241:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.157247:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.157249:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177406.157255:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177406.157258:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177406.157260:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306756864 00000100:00000010:2.0:1786177406.157265:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c840. 00000100:00000001:2.0:1786177406.157271:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1786177406.157278:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306756864 00000100:00000010:2.0:1786177406.157280:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e1290c6c0. 00000100:00000001:2.0:1786177406.157284:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1786177406.157287:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177406.157290:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.157292:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177406.157295:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177406/real 1786177406] req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177406.157303:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177406.157305:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177406.157306:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177406.157309:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177406.157310:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177406.157313:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177406.157326:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177406.157335:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.157341:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177406.157342:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.157344:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177406.157346:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177406.157349:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.157355:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177406.157386:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.157392:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177406.157395:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.157396:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177406.157398:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.157401:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177406.157403:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177406.157407:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177406.157411:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177406.157413:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177406.157417:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177406.157424:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235613440:192.168.202.132@tcp:250:kworker.0 00000100:00000001:2.0:1786177406.157429:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.157431:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177406.157433:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b9180 x1872930235613440/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.157439:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:1.0:1786177406.157449:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:2.0:1786177406.157452:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177406.157456:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177406.157460:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177406.157461:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177406.157463:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:2.0:1786177406.157465:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177406.157467:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af800. 02000000:00000001:2.0:1786177406.157477:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177406.157479:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177406.157481:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b9180. 00000100:00000001:2.0:1786177406.157484:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177406.157485:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.157487:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.157494:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177406.157496:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.157498:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177406.157500:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177406.157501:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1786177406.157504:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177406.157507:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177406.157510:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157515:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177406.157519:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177406.157522:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:1.0:1786177406.157525:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177406.157528:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:1.0:1786177406.157533:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:1.0:1786177406.157538:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000001:2.0:1786177406.157538:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177406.157541:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177406.157544:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 02000000:00000001:2.0:1786177406.157546:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1786177406.157548:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1786177406.157550:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1786177406.157552:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157554:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:1.0:1786177406.157557:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:1.0:1786177406.157559:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157561:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177406.157563:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1786177406.157566:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177406.157570:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177406.157572:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.157573:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177406.157577:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1786177406.157580:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613568:192.168.202.132@tcp:38:kworker.0 00000008:00000001:1.0:1786177406.157581:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.157586:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.157588:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1786177406.157591:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bad80. 02000000:00000001:2.0:1786177406.157593:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:1.0:1786177406.157595:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177406.157596:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177406.157600:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177406.157600:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1786177406.157603:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.157606:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:2.0:1786177406.157606:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:1.0:1786177406.157609:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177406.157610:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177406.157613:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177406.157617:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 02000000:00000010:1.0:1786177406.157619:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af000. 00000400:00000010:2.0:1786177406.157621:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000100:00000001:1.0:1786177406.157625:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1786177406.157627:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177406.157631:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235613568, portal 10 00000100:00100000:1.0:1786177406.157633:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177406.157634:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177406.157637:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487984768 : -110371221566848 : ffff9b9e328b8a80) 00000100:00000001:1.0:1786177406.157646:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1786177406.157647:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177406.157650:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177406.157658:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b748f214] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.157660:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177406.157666:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177406.157671:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000001:1.0:1786177406.157676:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1786177406.157676:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235613568, offset 0 00000100:00080000:1.0:1786177406.157681:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1786177406.157684:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000100:00080000:1.0:1786177406.157688:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1786177406.157690:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1786177406.157692:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00080000:1.0:1786177406.157693:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1786177406.157698:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1786177406.157700:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000100:00080000:1.0:1786177406.157702:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12080 00000400:00000200:2.0:1786177406.157707:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1786177406.157710:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177406.157714:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:1.0:1786177406.157717:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:1.0:1786177406.157720:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1786177406.157721:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1786177406.157726:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177406.157728:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157730:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177406.157736:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157740:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:1.0:1786177406.157742:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:1.0:1786177406.157743:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:1.0:1786177406.157744:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1786177406.157746:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:1.0:1786177406.157748:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000400:00000200:2.0:1786177406.157750:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426860:426860:256:4294967295] 192.168.202.132@tcp LPNI seq info [426822:426822:8:4294967295] 00000020:00000001:1.0:1786177406.157751:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:1.0:1786177406.157759:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000400:00000200:2.0:1786177406.157763:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:1.0:1786177406.157764:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1786177406.157769:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157773:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000400:00000200:2.0:1786177406.157778:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000020:00000040:1.0:1786177406.157779:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:1.0:1786177406.157785:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000800:00000200:2.0:1786177406.157790:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000100:00000001:1.0:1786177406.157791:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1786177406.157796:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:2.0:1786177406.157798:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 02000000:00000001:1.0:1786177406.157800:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1786177406.157804:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1786177406.157807:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:1.0:1786177406.157808:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1786177406.157811:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000800:00000200:2.0:1786177406.157813:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000008:00000001:1.0:1786177406.157816:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.157820:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1786177406.157821:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000010:1.0:1786177406.157825:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e328bbb80. 00000800:00000200:2.1:1786177406.157826:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000020:00000040:1.0:1786177406.157830:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 00000800:00000100:2.0:1786177406.157836:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 02000000:00000001:1.0:1786177406.157837:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000010:2.0:1786177406.157839:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 02000000:00000001:1.0:1786177406.157840:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.157845:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1786177406.157846:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1786177406.157847:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1786177406.157850:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177406.157854:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 02000000:00000010:1.0:1786177406.157858:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e261af800. 00000400:00000200:2.0:1786177406.157859:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177406.157862:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:1.0:1786177406.157865:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.157866:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:1.0:1786177406.157873:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1786177406.157874:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177406.157896:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177406.157898:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1786177406.157900:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177406.157904:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1786177406.157909:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3691ff3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177406.157917:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177406.157922:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12090) 00000100:00000001:2.0:1786177406.157948:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.157951:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177406.157955:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177406.157958:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:2.0:1786177406.157962:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177406.157964:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306753028 00000100:00000001:2.0:1786177406.157966:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306756864 00000100:00000001:2.0:1786177406.157968:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306753028 00000100:00000001:2.0:1786177406.157970:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306756864 00000100:00000001:2.0:1786177406.157972:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177406.157974:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.157976:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177406.157980:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177406/real 1786177406] req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177406.157987:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177406.157988:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177406.157989:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177406.157992:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177406.157993:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177406.157995:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177406.157998:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177406.158005:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158011:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177406.158013:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158014:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177406.158015:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177406.158019:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158024:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177406.158028:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158035:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177406.158037:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158038:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177406.158040:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158041:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177406.158043:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177406.158046:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177406.158049:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177406.158052:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177406.158055:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177406.158064:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613568:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177406.158068:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177406.158071:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158076:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177406.158078:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177406.158080:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177406.158081:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177406.158083:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177406.158086:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613696:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177406.158088:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177406.158090:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177406.158091:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177406.158093:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177406.158096:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177406.158098:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177406.158101:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177406.158103:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177406.158105:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177406.158108:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235613696, portal 10 00000100:00000001:2.0:1786177406.158110:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177406.158111:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487992832 : -110371221558784 : ffff9b9e328baa00) 00000100:00000040:2.0:1786177406.158116:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158121:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177406.158124:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177406.158126:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177406.158128:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235613696, offset 0 00000400:00000200:2.0:1786177406.158131:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177406.158135:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177406.158138:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177406.158141:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177406.158143:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177406.158145:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177406.158147:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177406.158149:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426861:426861:256:4294967295] 192.168.202.132@tcp LPNI seq info [426823:426823:8:4294967295] 00000400:00000200:2.0:1786177406.158153:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177406.158158:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177406.158162:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177406.158165:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177406.158169:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177406.158171:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786177406.158175:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177406.158178:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1786177406.158181:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177406.158183:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177406.158185:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177406.158188:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177406.158191:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177406.158193:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177406.158195:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177406.158199:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158207:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177406.158210:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158217:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.158219:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177406.158222:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177406.158225:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:2.0:1786177406.158228:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177406.158230:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306753028 00000100:00000001:2.0:1786177406.158232:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306756864 00000100:00000001:2.0:1786177406.158234:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306753028 00000100:00000001:2.0:1786177406.158236:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306756864 00000100:00000001:2.0:1786177406.158239:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177406.158241:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.158243:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177406.158247:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177406/real 1786177406] req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177406.158253:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177406.158254:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177406.158255:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177406.158258:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177406.158259:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177406.158262:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177406.158264:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177406.158273:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158278:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177406.158280:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158282:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177406.158283:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177406.158287:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158292:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177406.158297:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158302:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177406.158305:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158306:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177406.158307:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158309:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177406.158311:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177406.158409:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177406.158415:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177406.158417:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177406.158422:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177406.158431:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328baa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613696:192.168.202.132@tcp:38:kworker.0 00000100:00000001:2.0:1786177406.158435:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158437:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177406.158440:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328b8a80 x1872930235613568/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158447:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177406.158449:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177406.158451:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177406.158454:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177406.158456:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177406.158457:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177406.158459:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177406.158461:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261ad800. 02000000:00000001:2.0:1786177406.158463:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177406.158464:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177406.158466:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328b8a80. 00000100:00000001:2.0:1786177406.158468:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177406.158469:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158471:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177406.158474:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328baa00 x1872930235613696/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158480:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177406.158481:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177406.158482:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177406.158484:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177406.158485:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177406.158486:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:2.0:1786177406.158488:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177406.158489:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261aec00. 02000000:00000001:2.0:1786177406.158492:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177406.158493:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177406.158494:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328baa00. 00000100:00000001:2.0:1786177406.158497:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177406.158499:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158500:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158504:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177406.158506:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.158508:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177406.158509:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177406.158511:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177406.158514:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158521:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177406.158522:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177406.158524:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177406.158526:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177406.158528:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177406.158531:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613824:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177406.158549:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177406.158569:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177406.158571:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177406.158572:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177406.158576:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177406.158578:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177406.158581:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177406.158584:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177406.158588:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177406.158591:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235613824, portal 4 00000100:00000001:2.0:1786177406.158593:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177406.158594:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487993728 : -110371221557888 : ffff9b9e328bad80) 00000100:00000040:2.0:1786177406.158600:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158605:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177406.158607:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177406.158609:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177406.158611:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235613824, offset 0 00000400:00000200:2.0:1786177406.158615:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177406.158619:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177406.158622:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177406.158625:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177406.158627:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177406.158629:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177406.158631:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177406.158634:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426862:426862:256:4294967295] 192.168.202.132@tcp LPNI seq info [426824:426824:8:4294967295] 00000400:00000200:2.0:1786177406.158637:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177406.158643:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177406.158647:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177406.158650:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177406.158654:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177406.158656:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786177406.158660:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177406.158662:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1786177406.158666:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177406.158668:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177406.158671:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177406.158686:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177406.158689:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177406.158691:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177406.158693:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177406.158703:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158710:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177406.158713:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158718:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.158720:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177406.158723:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177406.158725:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:2.0:1786177406.158728:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177406.158729:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306753029 00000100:00000001:2.0:1786177406.158732:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306756864 00000100:00000001:2.0:1786177406.158733:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306753029 00000100:00000001:2.0:1786177406.158809:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306756864 00000100:00000001:2.0:1786177406.158813:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177406.158815:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.158817:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177406.158822:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177406/real 1786177406] req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177406.158830:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177406.158831:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177406.158833:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177406.158835:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177406.158837:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177406.158839:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177406.158842:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177406.158854:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158860:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177406.158861:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158862:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177406.158864:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177406.158867:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158872:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177406.158876:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158882:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177406.158884:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158885:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177406.158887:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.158888:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177406.158889:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177406.158891:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177406.158894:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177406.158896:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177406.158899:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177406.158905:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613824:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177406.158909:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1786177406.158912:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158924:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177406.158925:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177406.158927:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1786177406.158928:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177406.158930:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177406.158932:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613952:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177406.158935:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1786177406.158936:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1786177406.158938:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177406.158940:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1786177406.158943:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177406.158945:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1786177406.158948:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:2.0:1786177406.158951:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e100964d0. 00000400:00000010:2.0:1786177406.158953:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28f68. 00000100:00000200:2.0:1786177406.158955:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235613952, portal 4 00000100:00000001:2.0:1786177406.158957:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1786177406.158958:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702487997312 : -110371221554304 : ffff9b9e328bbb80) 00000100:00000040:2.0:1786177406.158963:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.158968:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1786177406.158970:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:2.0:1786177406.158972:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31c28660. 00000100:00000200:2.0:1786177406.158974:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235613952, offset 0 00000400:00000200:2.0:1786177406.158978:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:2.0:1786177406.158981:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177406.158984:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:2.0:1786177406.158988:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1786177406.158990:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:2.0:1786177406.158992:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1786177406.158994:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:2.0:1786177406.158997:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426863:426863:256:4294967295] 192.168.202.132@tcp LPNI seq info [426825:426825:8:4294967295] 00000400:00000200:2.0:1786177406.159000:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177406.159005:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:2.0:1786177406.159010:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177406.159013:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e12c6d800. 00000800:00000200:2.0:1786177406.159017:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.0:1786177406.159019:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1786177406.159023:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:2.1:1786177406.159025:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1786177406.159029:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:2.0:1786177406.159030:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e12c6d800. 00000400:00000200:2.0:1786177406.159033:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1786177406.159036:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1786177406.159039:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28660 00000400:00000010:2.0:1786177406.159040:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28660. 00000100:00000001:2.0:1786177406.159043:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1786177406.159046:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.159052:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177406.159054:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177431 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.159060:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.159061:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1786177406.159064:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177406.159067:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:2.0:1786177406.159069:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:2.0:1786177406.159071:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306753031 00000100:00000001:2.0:1786177406.159073:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306756864 00000100:00000001:2.0:1786177406.159075:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306753031 00000100:00000001:2.0:1786177406.159077:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306756864 00000100:00000001:2.0:1786177406.159079:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1786177406.159081:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.159083:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1786177406.159086:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177406/real 1786177406] req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1786177406.159093:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1786177406.159094:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1786177406.159096:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e100964d0. 00000400:00000200:2.0:1786177406.159098:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31c28f68 00000400:00000010:2.0:1786177406.159099:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31c28f68. 00000100:00000001:2.0:1786177406.159100:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1786177406.159104:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1786177406.159111:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.159116:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1786177406.159118:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.159120:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177406.159121:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1786177406.159124:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.159128:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177406.159132:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.159137:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1786177406.159139:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.159140:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1786177406.159141:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.159143:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1786177406.159144:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1786177406.159146:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1786177406.159149:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:2.0:1786177406.159151:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1786177406.159155:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1786177406.159162:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e328bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235613952:192.168.202.132@tcp:8:kworker.0 00000100:00000001:2.0:1786177406.159165:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.159166:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177406.159169:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bad80 x1872930235613824/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.159174:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177406.159176:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177406.159177:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:2.0:1786177406.159180:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177406.159181:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177406.159182:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177406.159184:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177406.159185:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af000. 02000000:00000001:2.0:1786177406.159187:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177406.159188:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177406.159189:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bad80. 00000100:00000001:2.0:1786177406.159193:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177406.159194:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.159195:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1786177406.159198:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e328bbb80 x1872930235613952/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177431 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177406.159204:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1786177406.159205:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1786177406.159206:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:2.0:1786177406.159208:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1786177406.159209:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:2.0:1786177406.159210:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:2.0:1786177406.159211:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:2.0:1786177406.159244:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e261af800. 02000000:00000001:2.0:1786177406.159247:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1786177406.159248:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1786177406.159249:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e328bbb80. 00000100:00000001:2.0:1786177406.159251:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1786177406.159252:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.159254:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177406.159258:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177406.159259:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177406.159261:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1786177406.159262:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177406.860920:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177406.860939:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177406.860942:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177406.860949:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177406.860952:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177406.860957:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177406.860960:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177406.860962:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177406.860965:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177407.053302:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177407.053352:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177407.053393:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177407.053395:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177407.053399:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177407.053401:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177407.155490:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177407.155496:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177407.155501:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177407.160952:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177407.160960:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177407.160962:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177407.160968:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177407.160972:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177407.160976:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177407.160978:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177407.160981:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177407.160983:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177407.884475:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177407.884483:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177407.884485:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177407.884490:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177407.884494:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177407.884498:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177407.884500:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177407.884503:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177407.884507:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177408.076684:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177408.076691:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177408.076697:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177408.076699:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177408.076701:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177408.076703:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177408.077504:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177408.077508:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177408.077513:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177408.204495:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177408.204503:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177408.204506:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177408.204512:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177408.204516:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177408.204520:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177408.204523:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177408.204526:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177408.204530:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177408.910385:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177408.910393:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177408.910395:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177408.910402:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177408.910405:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177408.910408:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177408.910411:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177408.910413:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177408.910416:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177409.100478:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177409.100487:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177409.100494:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177409.100495:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177409.100499:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177409.100500:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177409.101417:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177409.101422:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177409.101427:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177409.228472:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177409.228481:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177409.228483:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177409.228488:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177409.228491:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177409.228495:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177409.228497:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177409.228500:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177409.228502:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177409.934125:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177409.934134:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177409.934137:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177409.934143:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177409.934147:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177409.934151:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177409.934154:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177409.934158:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177409.934162:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177410.060626:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306756864, at 4306756928 00000100:00000010:0.0:1786177410.060642:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c840. 00000100:00000040:1.0:1786177410.062191:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306756864, at 4306756929 00000100:00000010:1.0:1786177410.062202:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e1290c6c0. 00010000:00000001:0.0:1786177410.124496:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177410.124503:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177410.124508:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177410.124510:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177410.124513:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177410.124514:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177410.124665:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177410.124669:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177410.124672:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177410.252861:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177410.252869:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177410.252871:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177410.252877:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177410.252881:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177410.252885:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177410.252887:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177410.252889:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177410.252892:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177410.961506:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177410.961514:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177410.961516:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177410.961522:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177410.961526:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177410.961530:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177410.961532:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177410.961535:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177410.961538:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177411.148503:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177411.148510:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177411.148516:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177411.148518:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177411.148522:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177411.148523:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177411.148626:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177411.148630:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177411.148633:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177411.276703:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177411.276710:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177411.276712:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177411.276718:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177411.276722:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177411.276725:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177411.276728:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177411.276730:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177411.276733:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177411.276815:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177411.276822:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177411.276824:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177411.276828:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177411.276832:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177411.276838:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12085 00000400:00000200:2.0:1786177411.276846:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177411.276852:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177411.276855:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177411.276858:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.276861:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177411.276863:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.276866:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177411.276869:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177411.276871:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177411.276874:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177411.276877:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177411.276879:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177411.276882:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177411.276885:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.276887:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177411.276888:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.276890:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177411.276893:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177411.276896:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177411.276899:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177411.276901:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177411.276904:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177411.276906:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.276908:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177411.276911:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177411.276919:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736a00. 00000020:00000040:2.0:1786177411.276922:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177411.276926:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177411.276928:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.276930:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177411.276933:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177411.276936:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177411.276942:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab000. 00000100:00000001:2.0:1786177411.276948:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177411.276967:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177411.276976:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177411.276980:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177411.276984:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009047bcf8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.277039:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177411.277042:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177411.277044:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177411.277051:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.277059:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177411.277060:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177411.277063:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177411.277066:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177411.277069:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177411.277072:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235614080:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177411.277076:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177411.277080:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177411.277082:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177411.277085:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177411.277114:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177411.277117:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177411.277123:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177411.277126:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177411.277132:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177411.277139:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235614080, portal 25 00000100:00000001:0.0:1786177411.277141:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177411.277143:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469626368 : -110371239925248 : ffff9b9e31736a00) 00000100:00000040:0.0:1786177411.277148:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.277156:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177411.277158:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177411.277161:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177411.277163:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235614080, offset 0 00000400:00000200:0.0:1786177411.277169:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177411.277175:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177411.277181:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177411.277186:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177411.277190:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177411.277194:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177411.277196:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177411.277201:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426864:426864:256:4294967295] 192.168.202.132@tcp LPNI seq info [426826:426826:8:4294967295] 00000400:00000200:0.0:1786177411.277205:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177411.277213:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177411.277219:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177411.277224:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177411.277230:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177411.277245:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177411.277251:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177411.277254:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177411.277259:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177411.277261:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177411.277278:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177411.277283:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177411.277288:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177411.277291:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177411.277295:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177411.277298:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177411.277596:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177411.277600:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177411.277603:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177411.277605:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177411.277606:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177411.277609:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177411.277612:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12085 00000400:00000200:2.0:1786177411.277616:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177411.277619:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177411.277622:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177411.277624:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.277627:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177411.277629:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.277631:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177411.277633:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177411.277634:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177411.277636:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177411.277637:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177411.277639:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177411.277641:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177411.277644:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.277645:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177411.277647:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.277649:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177411.277651:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177411.277654:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177411.277656:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177411.277658:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177411.277662:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177411.277664:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.277665:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177411.277670:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177411.277672:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.277674:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177411.277678:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31736680. 00000020:00000040:2.0:1786177411.277681:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177411.277684:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177411.277685:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.277687:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177411.277688:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177411.277691:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177411.277694:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa000. 00000100:00000001:2.0:1786177411.277698:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177411.277703:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177411.277711:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177411.277714:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177411.277718:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000047146bfe] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.277962:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177411.277968:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.277976:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.277977:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177411.277985:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177411.277989:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177411.277991:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306761985 00000100:00000010:0.0:1786177411.277996:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5ae0. 00000100:00000001:0.0:1786177411.278003:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177411.278009:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306761985 00000100:00000010:0.0:1786177411.278012:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5d20. 00000100:00000001:0.0:1786177411.278014:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177411.278017:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177411.278021:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.278023:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177411.278027:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177411/real 1786177411] req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177411.278036:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177411.278038:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177411.278040:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177411.278042:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177411.278044:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177411.278047:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177411.278050:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177411.278057:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278064:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177411.278066:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278068:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177411.278070:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177411.278074:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278080:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177411.278084:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278092:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177411.278095:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278096:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177411.278097:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278100:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177411.278101:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177411.278104:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177411.278108:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177411.278111:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177411.278115:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177411.278123:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235614080:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177411.278127:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278129:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177411.278132:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736a00 x1872930235614080/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278140:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177411.278142:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177411.278145:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177411.278148:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177411.278150:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177411.278151:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177411.278154:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177411.278157:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab000. 02000000:00000001:0.0:1786177411.278160:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177411.278162:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177411.278164:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736a00. 00000100:00000001:0.0:1786177411.278167:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177411.278168:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278170:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278176:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177411.278178:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.278180:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177411.278182:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177411.278183:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177411.278187:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278193:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177411.278195:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177411.278197:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177411.278199:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177411.278251:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177411.278254:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614208:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177411.278258:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177411.278260:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177411.278262:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177411.278271:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177411.278274:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177411.278276:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177411.278280:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177411.278283:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177411.278286:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177411.278289:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235614208, portal 10 00000100:00000001:0.0:1786177411.278291:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177411.278293:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469625472 : -110371239926144 : ffff9b9e31736680) 00000100:00000040:0.0:1786177411.278299:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278305:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177411.278308:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177411.278490:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177411.278494:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235614208, offset 0 00000400:00000200:0.0:1786177411.278500:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177411.278505:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177411.278509:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177411.278513:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177411.278516:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177411.278518:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177411.278520:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177411.278524:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426865:426865:256:4294967295] 192.168.202.132@tcp LPNI seq info [426827:426827:8:4294967295] 00000400:00000200:0.0:1786177411.278528:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177411.278534:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177411.278538:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177411.278541:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177411.278546:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177411.278549:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177411.278553:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177411.278556:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177411.278560:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177411.278562:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177411.278565:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177411.278569:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177411.278572:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177411.278574:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177411.278576:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177411.278580:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278588:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177411.278591:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278598:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.278599:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177411.278602:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177411.278605:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177411.278608:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177411.278610:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306758148 00000100:00000001:0.0:1786177411.278612:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306761985 00000100:00000001:0.0:1786177411.278614:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306758148 00000100:00000001:0.0:1786177411.278616:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306761985 00000100:00000001:0.0:1786177411.278618:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177411.278620:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.278622:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177411.278625:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177411/real 1786177411] req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177411.278638:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177411.278639:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177411.278641:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177411.278643:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177411.278644:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177411.278646:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177411.278649:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177411.278657:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278663:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177411.278665:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278667:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177411.278668:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177411.278671:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278677:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177411.278680:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278687:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177411.278689:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278691:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177411.278692:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278693:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177411.278695:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177411.278698:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177411.278700:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177411.278702:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177411.278707:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177411.278715:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31736680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614208:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177411.278718:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278720:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177411.278829:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31736680 x1872930235614208/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.278838:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177411.278839:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177411.278841:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177411.278844:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177411.278845:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177411.278847:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177411.278849:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177411.278851:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa000. 02000000:00000001:0.0:1786177411.278853:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177411.278854:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177411.278856:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31736680. 00000100:00000001:0.0:1786177411.278858:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177411.278859:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278861:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.278864:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177411.278866:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.278867:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177411.278869:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.279061:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177411.279065:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177411.279068:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177411.279070:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177411.279071:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177411.279074:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177411.279077:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12085 00000400:00000200:2.0:1786177411.279082:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177411.279085:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177411.279087:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177411.279090:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.279093:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177411.279095:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.279098:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177411.279100:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177411.279102:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177411.279104:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177411.279106:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177411.279108:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177411.279110:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177411.279113:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.279114:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177411.279116:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.279118:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177411.279121:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177411.279123:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177411.279125:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177411.279127:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177411.279131:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177411.279133:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.279135:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177411.279138:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177411.279140:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.279142:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177411.279146:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000020:00000040:2.0:1786177411.279149:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177411.279151:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177411.279153:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.279155:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177411.279157:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177411.279160:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177411.279164:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:2.0:1786177411.279173:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177411.279180:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177411.279193:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177411.279196:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177411.279199:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177411.279207:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177411.279210:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177411.279213:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177411.279214:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177411.279216:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177411.279219:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177411.279221:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12085 00000400:00000200:2.0:1786177411.279225:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177411.279227:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177411.279229:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1786177411.279461:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177411.279464:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177411.279466:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177411.279471:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.279478:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177411.279480:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177411.279482:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177411.279484:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177411.279486:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177411.279489:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614336:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177411.279493:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177411.279495:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177411.279496:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177411.279498:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177411.279501:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177411.279503:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177411.279507:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177411.279511:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177411.279514:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177411.279518:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235614336, portal 10 00000100:00000001:0.0:1786177411.279520:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177411.279521:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469623680 : -110371239927936 : ffff9b9e31735f80) 00000100:00000040:0.0:1786177411.279528:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.279535:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177411.279538:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177411.279540:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177411.279543:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235614336, offset 0 00000400:00000200:0.0:1786177411.279548:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177411.279553:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177411.279557:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177411.279560:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177411.279564:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177411.279566:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177411.279568:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177411.279572:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426866:426866:256:4294967295] 192.168.202.132@tcp LPNI seq info [426828:426828:8:4294967295] 00000400:00000200:0.0:1786177411.279575:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177411.279581:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177411.279585:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177411.279589:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177411.279593:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177411.279596:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177411.279600:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177411.279603:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177411.279607:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177411.279608:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177411.279612:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177411.279615:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177411.279618:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177411.279620:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177411.279622:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177411.279626:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.279633:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177411.279636:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.279642:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.279644:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177411.279647:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177411.279650:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177411.279653:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177411.279655:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306758150 00000100:00000001:0.0:1786177411.279657:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306761985 00000100:00000001:0.0:1786177411.279659:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306758150 00000100:00000001:0.0:1786177411.279661:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306761985 00000100:00000001:0.0:1786177411.279663:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177411.279665:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.279666:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177411.279670:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177411/real 1786177411] req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177411.279682:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177411.279683:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177411.279685:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177411.279687:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177411.279688:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177411.279690:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177411.279693:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177411.279700:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.279706:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177411.279708:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.279709:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177411.279711:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177411.279714:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.279720:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1786177411.279860:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.279862:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177411.279864:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.279866:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177411.279868:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177411.279870:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177411.279871:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177411.279874:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177411.279876:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177411.279878:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177411.279881:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.279882:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177411.279884:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.279886:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177411.279908:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177411.279912:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177411.279914:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177411.279916:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177411.279919:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177411.279921:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.279923:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177411.279926:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177411.279929:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.279931:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177411.279934:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000020:00000040:2.0:1786177411.279937:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177411.279939:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177411.279940:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.279943:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177411.279944:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177411.279946:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177411.279950:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000040:0.0:1786177411.280024:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.280033:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177411.280035:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.280037:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177411.280038:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.280040:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177411.280042:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177411.280045:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177411.280049:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177411.280051:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177411.280056:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177411.280064:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614336:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177411.280068:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.280070:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177411.280072:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735f80 x1872930235614336/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.280078:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177411.280079:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177411.280081:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177411.280084:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177411.280086:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177411.280087:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177411.280089:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177411.280091:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:0.0:1786177411.280094:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177411.280095:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177411.280097:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735f80. 00000100:00000001:0.0:1786177411.280099:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177411.280100:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.280102:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.280107:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177411.280108:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.280110:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177411.280112:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.280149:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177411.280156:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177411.280164:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177411.280166:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177411.280170:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177411.280188:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177411.280191:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177411.280195:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177411.280196:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177411.280198:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177411.280200:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177411.280203:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12085 00000400:00000200:2.0:1786177411.280207:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177411.280211:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177411.280213:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177411.280216:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.280217:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177411.280219:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.280221:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177411.280224:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177411.280225:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177411.280226:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177411.280228:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177411.280231:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177411.280480:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177411.280484:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.280487:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177411.280488:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177411.280490:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177411.280492:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177411.280495:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177411.280496:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177411.280498:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177411.280501:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177411.280503:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.280505:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177411.280506:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177411.280508:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.280510:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177411.280513:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000020:00000040:2.0:1786177411.280515:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177411.280517:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177411.280535:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.280537:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177411.280538:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177411.280541:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177411.280545:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786177411.280549:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177411.280554:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177411.280562:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177411.280564:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177411.280566:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177411.280574:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177411.280579:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12095) 00000100:00000001:0.0:1786177411.280630:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177411.280632:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177411.280634:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177411.280638:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.280646:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177411.280647:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177411.280650:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177411.280652:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177411.280655:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177411.280658:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614464:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177411.280661:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177411.280663:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177411.280665:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177411.280666:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177411.280669:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177411.280671:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177411.280675:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177411.280678:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177411.280682:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177411.280687:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235614464, portal 4 00000100:00000001:0.0:1786177411.280689:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177411.280690:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469628160 : -110371239923456 : ffff9b9e31737100) 00000100:00000040:0.0:1786177411.280696:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.280702:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177411.280704:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177411.280706:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177411.280708:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235614464, offset 0 00000400:00000200:0.0:1786177411.280713:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177411.280717:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177411.280720:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177411.281846:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177411.281853:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177411.281857:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177411.281860:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177411.281864:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426867:426867:256:4294967295] 192.168.202.132@tcp LPNI seq info [426829:426829:8:4294967295] 00000400:00000200:0.0:1786177411.281869:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177411.281877:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177411.281883:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177411.281890:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177411.281896:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177411.281900:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177411.281905:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177411.281908:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177411.281912:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177411.281914:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177411.281919:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177411.281922:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177411.281927:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177411.281929:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177411.281933:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177411.281937:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.281948:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177411.281951:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282019:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.282021:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177411.282027:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177411.282031:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177411.282035:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177411.282038:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306758152 00000100:00000001:0.0:1786177411.282041:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306761985 00000100:00000001:0.0:1786177411.282043:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306758152 00000100:00000001:0.0:1786177411.282045:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306761985 00000100:00000001:0.0:1786177411.282047:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177411.282049:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.282051:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177411.282056:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177411/real 1786177411] req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177411.282071:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177411.282073:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177411.282075:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177411.282079:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177411.282080:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177411.282083:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177411.282086:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177411.282094:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282100:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177411.282102:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.282104:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177411.282106:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177411.282109:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282116:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177411.282120:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282126:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177411.282129:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.282130:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177411.282131:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.282133:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177411.282135:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177411.282138:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177411.282142:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177411.282145:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177411.282149:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177411.282155:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614464:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177411.282158:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177411.282160:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282167:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177411.282168:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177411.282171:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177411.282174:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177411.282176:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177411.282179:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614592:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177411.282181:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177411.282184:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177411.282185:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177411.282187:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177411.282190:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177411.282191:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177411.282194:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177411.282197:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177411.282201:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177411.282206:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235614592, portal 4 00000100:00000001:0.0:1786177411.282208:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177411.282209:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469618304 : -110371239933312 : ffff9b9e31734a80) 00000100:00000040:0.0:1786177411.282213:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282218:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177411.282219:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177411.282221:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177411.282223:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235614592, offset 0 00000400:00000200:0.0:1786177411.282228:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177411.282331:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177411.282338:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177411.282341:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177411.282344:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177411.282379:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177411.282397:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177411.282400:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426868:426868:256:4294967295] 192.168.202.132@tcp LPNI seq info [426830:426830:8:4294967295] 00000400:00000200:0.0:1786177411.282405:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177411.282420:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177411.282424:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177411.282427:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177411.282441:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177411.282443:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1786177411.282446:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177411.282448:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 22 secs later) 00000800:00000100:0.0:1786177411.282450:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177411.282451:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177411.282453:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177411.282456:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177411.282458:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177411.282467:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177411.282469:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177411.282472:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282477:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177411.282479:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177436 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282491:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.282492:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177411.282494:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177411.282496:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177411.282498:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177411.282500:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306758155 00000100:00000001:0.0:1786177411.282502:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306761985 00000100:00000001:0.0:1786177411.282503:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306758155 00000100:00000001:0.0:1786177411.282505:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306761985 00000100:00000001:0.0:1786177411.282506:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177411.282507:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.282516:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177411.282519:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177411/real 1786177411] req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177411.282524:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177411.282525:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177411.282526:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177411.282527:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177411.282529:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177411.282530:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177411.282532:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177411.282536:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282547:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177411.282548:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.282549:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177411.282550:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177411.282552:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282556:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177411.282572:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282576:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177411.282578:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.282579:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177411.282580:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.282581:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177411.282582:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177411.282584:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177411.282586:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177411.282587:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177411.282589:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177411.282593:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614592:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177411.282596:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.282597:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177411.282599:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737100 x1872930235614464/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282603:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177411.282604:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177411.282606:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177411.282608:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177411.282609:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177411.282615:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177411.282617:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177411.282619:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:0.0:1786177411.282621:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177411.282622:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177411.282624:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737100. 00000100:00000001:0.0:1786177411.282626:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177411.282626:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.282628:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177411.282629:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734a80 x1872930235614592/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177436 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177411.282637:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177411.282637:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177411.282638:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177411.282640:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177411.282640:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177411.282641:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177411.282642:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177411.282642:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:0.0:1786177411.282644:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177411.282644:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177411.282645:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734a80. 00000100:00000001:0.0:1786177411.282646:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177411.282647:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.282648:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177411.282652:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177411.282653:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177411.282655:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177411.282656:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177411.981961:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177411.981968:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177411.981970:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177411.981976:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177411.981980:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177411.981983:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177411.981985:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177411.981988:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177411.981991:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177412.172462:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177412.172469:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177412.172475:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177412.172478:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177412.172481:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177412.172484:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177412.172487:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177412.172489:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177412.172491:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177412.172493:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177412.172496:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177412.172497:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177412.172585:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177412.172588:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177412.172591:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177412.300472:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177412.300480:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177412.300483:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177412.300500:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177412.300504:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177412.300508:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177412.300511:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177412.300514:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177412.300517:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177413.004486:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177413.004502:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177413.004505:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177413.004511:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177413.004515:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177413.004518:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177413.004521:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177413.004523:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177413.004527:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177413.196494:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177413.196502:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177413.196509:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177413.196510:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177413.196514:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177413.196515:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177413.196699:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177413.196703:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177413.196706:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177413.451771:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177413.451778:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177413.451781:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177413.451785:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177413.451788:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177413.451792:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177413.451795:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177413.451797:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177413.451800:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177414.028447:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1786177414.028455:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1786177414.028457:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177414.028463:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:2.0:1786177414.028467:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1786177414.028470:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177414.028473:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1786177414.028475:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1786177414.028479:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177414.221277:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177414.221287:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177414.221289:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177414.221292:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177414.221298:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177414.221300:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177414.221302:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177414.221305:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177414.221308:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177414.221310:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177414.221313:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177414.221315:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177414.221316:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177414.221318:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177414.221319:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177414.221320:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177414.221322:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177414.221324:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177414.221559:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177414.221563:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177414.221565:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1786177414.607640:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1786177414.607647:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1786177414.607649:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177414.607654:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:3.0:1786177414.607657:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1786177414.607661:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177414.607663:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1786177414.607666:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177414.607669:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177415.062997:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177415.063005:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177415.063008:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177415.063015:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177415.063019:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177415.063023:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177415.063026:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177415.063028:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177415.063031:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177415.180509:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306761985, at 4306762048 00000100:00000010:0.0:1786177415.180526:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5ae0. 00000100:00000040:1.0:1786177415.180668:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306761985, at 4306762048 00000100:00000010:1.0:1786177415.180675:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5d20. 00010000:00000001:0.0:1786177415.245431:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177415.245438:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177415.245444:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177415.245445:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177415.245448:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177415.245450:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177415.245606:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177415.245610:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177415.245613:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177415.628588:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177415.628595:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177415.628597:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177415.628602:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177415.628606:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177415.628609:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177415.628612:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177415.628614:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177415.628618:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177416.076519:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177416.076528:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177416.076530:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177416.076535:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177416.076539:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177416.076542:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177416.076545:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177416.076547:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177416.076550:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177416.268485:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177416.268492:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177416.268498:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177416.268499:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177416.268502:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177416.268503:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177416.268604:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177416.268608:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177416.268610:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1786177416.396954:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177416.396965:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1786177416.396968:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177416.396972:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177416.396976:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177416.396983:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12090 00000400:00000200:2.0:1786177416.396993:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177416.396999:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177416.397002:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177416.397005:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397010:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177416.397012:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397015:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177416.397019:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177416.397021:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:2.0:1786177416.397024:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177416.397027:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:2.0:1786177416.397030:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:2.0:1786177416.397032:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:2.0:1786177416.397035:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397037:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177416.397039:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397041:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177416.397044:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:2.0:1786177416.397047:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397049:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177416.397053:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177416.397056:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177416.397059:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397061:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1786177416.397064:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177416.397075:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735180. 00000020:00000040:2.0:1786177416.397078:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:2.0:1786177416.397082:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177416.397084:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397087:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177416.397090:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177416.397094:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177416.397101:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab400. 00000100:00000001:2.0:1786177416.397109:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177416.397118:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177416.397128:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177416.397132:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177416.397136:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ba0f078] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177416.397163:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177416.397166:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177416.397170:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1786177416.397171:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177416.397172:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177416.397175:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177416.397178:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12090 00000400:00000200:2.0:1786177416.397181:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177416.397184:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177416.397186:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177416.397188:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397190:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177416.397193:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397195:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177416.397197:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177416.397198:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:2.0:1786177416.397200:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177416.397202:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:2.0:1786177416.397204:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:2.0:1786177416.397206:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:2.0:1786177416.397209:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397210:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177416.397212:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397214:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177416.397246:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:2.0:1786177416.397249:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397251:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177416.397253:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177416.397258:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177416.397260:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397261:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177416.397265:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177416.397268:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397270:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177416.397273:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734e00. 00000020:00000040:2.0:1786177416.397276:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177416.397278:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177416.397279:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397281:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177416.397283:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177416.397284:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177416.397288:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa8000. 00000100:00000001:2.0:1786177416.397291:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177416.397296:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177416.397304:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177416.397306:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177416.397310:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000001f62212] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177416.397318:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177416.397320:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177416.397324:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1786177416.397325:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177416.397327:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177416.397329:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177416.397331:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12090 00000400:00000200:2.0:1786177416.397335:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177416.397338:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177416.397340:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177416.397343:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397345:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177416.397347:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397349:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177416.397351:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177416.397352:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:2.0:1786177416.397354:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177416.397355:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:2.0:1786177416.397388:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:2.0:1786177416.397390:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:2.0:1786177416.397393:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397395:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177416.397397:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397399:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177416.397402:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:2.0:1786177416.397404:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397407:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177416.397409:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177416.397411:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177416.397413:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397415:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177416.397417:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1786177416.397419:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397421:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177416.397426:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31734a80. 00000020:00000040:2.0:1786177416.397428:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:2.0:1786177416.397430:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177416.397431:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397433:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177416.397434:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177416.397436:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177416.397440:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aa9400. 00000100:00000001:2.0:1786177416.397443:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177416.397448:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177416.397455:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177416.397457:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177416.397461:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000016b304] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177416.397468:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177416.397470:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177416.397473:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1786177416.397475:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177416.397477:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177416.397479:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177416.397482:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12090 00000400:00000200:2.0:1786177416.397485:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177416.397488:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177416.397490:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177416.397493:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397494:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177416.397496:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397498:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177416.397500:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177416.397502:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:2.0:1786177416.397503:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177416.397505:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:2.0:1786177416.397507:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:2.0:1786177416.397509:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:2.0:1786177416.397512:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397514:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177416.397515:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397517:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177416.397519:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:2.0:1786177416.397521:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397523:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177416.397525:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177416.397527:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177416.397529:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397531:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177416.397533:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177416.397535:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397537:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177416.397540:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31737100. 00000020:00000040:2.0:1786177416.397542:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177416.397544:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177416.397546:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397548:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177416.397549:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177416.397551:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177416.397554:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aaa400. 00000100:00000001:2.0:1786177416.397557:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177416.397561:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177416.397568:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177416.397570:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177416.397573:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3f725a9] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177416.397580:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1786177416.397582:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1786177416.397585:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1786177416.397587:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1786177416.397588:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1786177416.397590:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1786177416.397592:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12090 00000400:00000200:2.0:1786177416.397596:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:2.0:1786177416.397599:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:2.0:1786177416.397601:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1786177416.397603:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397605:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177416.397607:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397608:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:2.0:1786177416.397610:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:2.0:1786177416.397611:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:2.0:1786177416.397613:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1786177416.397614:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:2.0:1786177416.397616:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:2.0:1786177416.397619:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:2.0:1786177416.397621:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397623:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1786177416.397624:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397626:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:2.0:1786177416.397629:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:2.0:1786177416.397631:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:2.0:1786177416.397633:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1786177416.397635:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1786177416.397637:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1786177416.397639:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397641:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1786177416.397643:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1786177416.397645:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397647:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1786177416.397650:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e31735f80. 00000020:00000040:2.0:1786177416.397653:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:2.0:1786177416.397655:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1786177416.397656:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1786177416.397658:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1786177416.397659:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1786177416.397661:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1786177416.397664:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e31aab800. 00000100:00000001:2.0:1786177416.397667:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1786177416.397671:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1786177416.397678:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1786177416.397680:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177416.397683:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000133928b5] to pc [ptlrpcd_rcv+-1] req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1786177416.397690:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1786177416.397695:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12100) 00000100:00000001:0.0:1786177416.399400:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177416.399408:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177416.399411:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177416.399421:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.399437:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177416.399439:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177416.399443:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177416.399446:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177416.399449:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177416.399452:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235614720:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177416.399457:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177416.399461:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177416.399463:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177416.399466:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177416.399472:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177416.399475:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177416.399481:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177416.399485:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177416.399492:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177416.399499:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235614720, portal 25 00000100:00000001:0.0:1786177416.399501:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177416.399503:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469620096 : -110371239931520 : ffff9b9e31735180) 00000100:00000040:0.0:1786177416.399508:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.399516:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177416.399519:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177416.399521:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177416.399524:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235614720, offset 0 00000400:00000200:0.0:1786177416.399531:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177416.399539:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177416.399546:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177416.399551:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177416.399556:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177416.399560:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177416.399563:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177416.399567:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426869:426869:256:4294967295] 192.168.202.132@tcp LPNI seq info [426831:426831:8:4294967295] 00000400:00000200:0.0:1786177416.399572:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177416.399582:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177416.399587:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177416.399592:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177416.399599:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177416.399603:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177416.399609:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177416.399612:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177416.399617:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177416.399619:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177416.399626:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177416.399631:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177416.399636:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177416.399638:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177416.399642:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177416.399646:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.399657:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177416.399660:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.399667:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177416.399669:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177416.399674:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177416.399678:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177416.399681:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306767107 00000100:00000010:0.0:1786177416.399686:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5ae0. 00000100:00000001:0.0:1786177416.399693:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177416.399701:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306767107 00000100:00000010:0.0:1786177416.399704:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5f00. 00000100:00000001:0.0:1786177416.399707:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177416.399710:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177416.399713:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177416.399715:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177416.399720:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177416/real 1786177416] req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177416.399728:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177416.399730:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177416.399732:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177416.399734:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177416.399736:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177416.399738:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177416.399741:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177416.399750:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.399756:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177416.399758:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.399760:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177416.399762:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177416.399766:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.399773:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177416.399777:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.399785:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177416.399788:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.399789:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177416.399790:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.399793:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177416.399795:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177416.399798:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177416.399802:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177416.399805:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177416.399809:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177416.399818:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235614720:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177416.399823:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177416.399841:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.399848:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177416.399849:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177416.399852:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177416.399854:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177416.399856:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177416.399859:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614848:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177416.399862:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177416.399864:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177416.399866:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177416.399867:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177416.399870:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177416.399872:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177416.399876:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177416.399879:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177416.399882:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177416.399885:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235614848, portal 10 00000100:00000001:0.0:1786177416.399887:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177416.399888:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469619200 : -110371239932416 : ffff9b9e31734e00) 00000100:00000040:0.0:1786177416.399896:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.399902:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177416.399904:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177416.399906:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177416.399909:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235614848, offset 0 00000400:00000200:0.0:1786177416.399913:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177416.399916:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177416.399919:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177416.399922:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177416.399925:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177416.399927:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177416.399929:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177416.399932:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426870:426870:256:4294967295] 192.168.202.132@tcp LPNI seq info [426832:426832:8:4294967295] 00000400:00000200:0.0:1786177416.399936:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177416.399942:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177416.399946:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177416.399949:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177416.399953:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177416.399955:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177416.399959:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177416.399962:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177416.399965:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177416.399967:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177416.399969:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177416.399973:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177416.399975:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177416.399977:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177416.399979:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177416.399982:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.399990:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177416.399992:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.399998:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177416.400000:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177416.400002:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177416.400007:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177416.400009:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177416.400011:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306763270 00000100:00000001:0.0:1786177416.400014:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306767107 00000100:00000001:0.0:1786177416.400016:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306763270 00000100:00000001:0.0:1786177416.400018:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306767107 00000100:00000001:0.0:1786177416.400020:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177416.400022:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177416.400024:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177416.400027:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177416/real 1786177416] req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177416.400034:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177416.400035:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177416.400036:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177416.400039:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177416.400040:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177416.400042:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177416.400045:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177416.400053:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.400058:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177416.400060:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.400061:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177416.400063:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177416.400066:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.400072:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177416.400075:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.400082:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177416.400084:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.400086:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177416.400087:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.400089:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177416.400091:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177416.400094:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177416.400097:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177416.400099:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177416.400103:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177416.400111:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614848:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177416.400114:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177416.400117:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.400123:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177416.400124:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177416.400126:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177416.400127:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177416.400129:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177416.400132:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614976:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177416.400135:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177416.400137:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177416.400139:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177416.400140:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177416.400143:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177416.400145:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177416.400149:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177416.400151:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177416.400154:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177416.400156:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235614976, portal 10 00000100:00000001:0.0:1786177416.400158:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177416.400159:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469618304 : -110371239933312 : ffff9b9e31734a80) 00000100:00000040:0.0:1786177416.400165:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.400170:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177416.400172:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177416.400174:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177416.400176:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235614976, offset 0 00000400:00000200:0.0:1786177416.400180:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177416.400183:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177416.400187:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177416.400189:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177416.400191:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177416.400194:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177416.400196:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177416.400198:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426871:426871:256:4294967295] 192.168.202.132@tcp LPNI seq info [426833:426833:8:4294967295] 00000400:00000200:0.0:1786177416.400202:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177416.400208:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177416.400212:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177416.401947:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177416.401958:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177416.401962:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177416.401967:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177416.401970:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177416.401973:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177416.401976:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177416.401979:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177416.401984:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177416.401990:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177416.401993:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177416.401997:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177416.402003:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.402014:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177416.402017:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.402024:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177416.402026:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177416.402031:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177416.402036:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177416.402040:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177416.402043:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306763272 00000100:00000001:0.0:1786177416.402046:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306767107 00000100:00000001:0.0:1786177416.402048:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306763272 00000100:00000001:0.0:1786177416.402050:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306767107 00000100:00000001:0.0:1786177416.402052:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177416.402054:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177416.402057:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177416.402061:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177416/real 1786177416] req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177416.402070:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177416.402071:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177416.402074:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177416.402076:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177416.402078:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177416.402080:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177416.402083:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177416.402091:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.402097:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177416.402099:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.402101:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177416.402103:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177416.402106:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.402113:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177416.402117:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.402122:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177416.402125:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.402126:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177416.402127:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.402129:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177416.402131:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177416.402134:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177416.402138:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177416.402140:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177416.402144:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177416.402150:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31734a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235614976:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177416.402155:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177416.402159:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.402166:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177416.402168:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177416.402170:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177416.402173:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177416.402175:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177416.402178:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615104:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177416.402182:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177416.402184:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177416.402186:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177416.402188:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177416.402192:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177416.402194:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177416.402198:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177416.402202:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177416.402206:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177416.402210:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235615104, portal 4 00000100:00000001:0.0:1786177416.402213:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177416.402909:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469628160 : -110371239923456 : ffff9b9e31737100) 00000100:00000040:0.0:1786177416.402920:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.402927:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177416.402929:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177416.402934:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177416.402937:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235615104, offset 0 00000400:00000200:0.0:1786177416.402942:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177416.402949:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177416.402955:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177416.402959:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177416.402963:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177416.402966:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177416.402968:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177416.402972:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426872:426872:256:4294967295] 192.168.202.132@tcp LPNI seq info [426834:426834:8:4294967295] 00000400:00000200:0.0:1786177416.402976:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177416.402984:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177416.402989:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177416.402992:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177416.402997:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177416.403000:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177416.403004:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177416.403006:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177416.403036:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177416.403038:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177416.403041:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177416.403045:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177416.403048:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177416.403049:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177416.403052:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177416.403055:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.403064:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177416.403068:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.403074:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177416.403075:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177416.403079:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177416.403082:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177416.403085:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177416.403087:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306763272 00000100:00000001:0.0:1786177416.403090:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306767107 00000100:00000001:0.0:1786177416.403092:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306763272 00000100:00000001:0.0:1786177416.403094:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306767107 00000100:00000001:0.0:1786177416.403096:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177416.403098:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177416.403100:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177416.403104:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177416/real 1786177416] req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177416.403112:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177416.403114:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177416.403115:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177416.403117:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177416.403119:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177416.403121:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177416.403124:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177416.403132:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.403138:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177416.403140:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.403142:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177416.403143:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177416.403147:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.403153:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177416.403157:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.403164:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177416.403166:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.403168:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177416.403169:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.403171:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177416.403172:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177416.403175:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177416.403179:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177416.403181:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177416.403185:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177416.403192:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31737100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615104:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177416.403196:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177416.403200:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.403206:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177416.403207:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177416.403209:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177416.403211:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177416.403213:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177416.404002:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615232:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177416.404007:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177416.404010:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177416.404013:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177416.404015:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177416.404019:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177416.404022:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177416.404026:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177416.404029:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177416.404034:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177416.404038:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235615232, portal 4 00000100:00000001:0.0:1786177416.404040:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177416.404041:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702469623680 : -110371239927936 : ffff9b9e31735f80) 00000100:00000040:0.0:1786177416.404048:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.404055:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177416.404057:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177416.404059:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177416.404062:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235615232, offset 0 00000400:00000200:0.0:1786177416.404066:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177416.404071:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177416.404075:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177416.404079:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177416.404081:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177416.404084:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177416.404086:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177416.404089:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426873:426873:256:4294967295] 192.168.202.132@tcp LPNI seq info [426835:426835:8:4294967295] 00000400:00000200:0.0:1786177416.404092:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177416.404099:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177416.404103:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177416.404107:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177416.404111:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177416.404113:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1786177416.404118:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177416.404121:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 17 secs later) 00000800:00000100:0.0:1786177416.404124:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177416.404126:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177416.404130:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177416.404133:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177416.404136:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177416.404138:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177416.404141:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177416.404144:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.404152:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177416.404155:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177441 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.404160:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177416.404162:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177416.404165:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177416.404169:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177416.404172:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177416.404174:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306763276 00000100:00000001:0.0:1786177416.404176:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306767107 00000100:00000001:0.0:1786177416.404179:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306763276 00000100:00000001:0.0:1786177416.404181:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306767107 00000100:00000001:0.0:1786177416.404183:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177416.404185:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177416.404187:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177416.404190:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177416/real 1786177416] req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177416.404198:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177416.404199:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177416.404201:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177416.404203:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177416.404204:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177416.404206:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177416.404209:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177416.404908:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.404916:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177416.404918:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.404920:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177416.404922:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177416.404926:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.404933:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177416.404936:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.404942:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177416.404945:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.404946:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177416.404947:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.404949:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177416.404951:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177416.404954:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177416.404958:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177416.404961:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177416.404964:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177416.404972:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e31735f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615232:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177416.404976:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.404979:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177416.404982:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735180 x1872930235614720/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.404989:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177416.404992:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177416.404994:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177416.404998:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177416.405001:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177416.405002:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177416.405004:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177416.405007:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab400. 02000000:00000001:0.0:1786177416.405011:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177416.405013:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177416.405014:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735180. 00000100:00000001:0.0:1786177416.405017:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177416.405018:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.405020:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177416.405023:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734e00 x1872930235614848/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.405034:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177416.405036:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177416.405037:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177416.405039:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177416.405040:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177416.405042:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177416.405043:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177416.405045:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa8000. 02000000:00000001:0.0:1786177416.405047:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177416.405048:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177416.405049:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734e00. 00000100:00000001:0.0:1786177416.405051:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177416.405052:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.405054:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177416.405057:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31734a80 x1872930235614976/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.405064:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177416.405065:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177416.405066:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177416.405068:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177416.405069:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177416.405070:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177416.405072:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177416.405073:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aa9400. 02000000:00000001:0.0:1786177416.405075:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177416.405077:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177416.405078:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31734a80. 00000100:00000001:0.0:1786177416.405080:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177416.405081:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.405083:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177416.405086:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31737100 x1872930235615104/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.405092:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177416.405093:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177416.405095:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177416.405097:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177416.405098:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177416.405099:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177416.405101:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177416.405102:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aaa400. 02000000:00000001:0.0:1786177416.405104:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177416.405105:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177416.405106:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31737100. 00000100:00000001:0.0:1786177416.405108:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177416.405109:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.405110:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177416.405113:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e31735f80 x1872930235615232/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177441 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177416.405119:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177416.405120:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177416.405122:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177416.405123:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177416.405124:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177416.405125:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177416.405127:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177416.405128:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e31aab800. 02000000:00000001:0.0:1786177416.405130:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177416.405131:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177416.405132:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e31735f80. 00000100:00000001:0.0:1786177416.405134:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177416.405135:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.405137:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177416.405144:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177416.405146:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177416.405148:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177416.405150:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177416.656885:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177416.656893:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177416.656896:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177416.656902:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177416.656961:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177416.656965:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177416.656967:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177416.656969:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177416.656972:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177417.100767:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177417.100775:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177417.100777:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177417.100782:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177417.100786:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177417.100790:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177417.100793:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177417.100795:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177417.100798:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177417.292629:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177417.292636:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177417.292642:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177417.292643:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177417.292647:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177417.292648:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177417.292851:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177417.292855:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177417.292858:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177417.676729:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177417.676736:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177417.676738:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177417.676743:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177417.676746:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177417.676749:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177417.676752:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177417.676754:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177417.676757:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177418.124633:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177418.124640:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177418.124642:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177418.124648:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177418.124651:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177418.124655:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177418.124657:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177418.124659:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177418.124662:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177418.316611:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177418.316618:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177418.316626:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177418.316627:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177418.316630:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177418.316632:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177418.386339:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177418.386346:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177418.386350:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177418.700497:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177418.700506:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177418.700508:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177418.700515:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177418.700520:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177418.700524:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177418.700527:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177418.700530:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177418.700533:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177419.148487:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177419.148495:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177419.148498:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177419.148504:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177419.148508:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177419.148511:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177419.148513:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177419.148515:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177419.148518:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177419.340798:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177419.340804:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177419.340810:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177419.340811:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177419.340814:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177419.340815:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177419.340911:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177419.340916:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177419.340918:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177419.724721:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177419.724729:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177419.724731:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177419.724737:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177419.724741:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177419.724753:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177419.724756:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177419.724758:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177419.724761:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177420.172568:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177420.172576:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177420.172578:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177420.172583:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177420.172587:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177420.172591:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177420.172593:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177420.172596:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177420.172599:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177420.300489:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306767107, at 4306767168 00000100:00000010:0.0:1786177420.300505:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5ae0. 00000100:00000040:1.0:1786177420.300652:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306767107, at 4306767168 00000100:00000010:1.0:1786177420.300658:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5f00. 00010000:00000001:0.0:1786177420.364692:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177420.364699:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177420.364705:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177420.364706:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177420.364709:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177420.364710:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177420.372093:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177420.372101:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177420.372106:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177420.753166:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177420.753182:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177420.753185:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177420.753190:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177420.753194:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177420.753198:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177420.753200:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177420.753203:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177420.753206:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177421.197397:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177421.197404:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177421.197406:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177421.197411:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177421.197415:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177421.197419:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177421.197421:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177421.197424:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177421.197427:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177421.388687:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177421.388693:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177421.388698:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177421.388699:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177421.388702:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177421.388703:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177421.388855:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177421.388858:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177421.388861:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177421.518862:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177421.518873:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177421.518876:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177421.518879:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177421.518883:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177421.518890:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12095 00000400:00000200:3.0:1786177421.518899:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177421.518905:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177421.518908:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177421.518911:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.518915:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177421.518917:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.518919:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177421.518923:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177421.518925:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177421.518927:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177421.518930:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177421.518933:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177421.518935:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177421.518938:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.518940:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177421.518943:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.518945:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177421.518947:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177421.518951:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177421.518953:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177421.518956:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177421.518959:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177421.518962:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.518964:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177421.518966:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177421.518973:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325f80. 00000020:00000040:3.0:1786177421.518976:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177421.518980:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177421.518982:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.518985:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177421.518987:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177421.518991:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177421.518998:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa000. 00000100:00000001:3.0:1786177421.519003:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177421.519013:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177421.519023:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177421.519027:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177421.519032:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000037aa943] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177421.519055:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177421.519058:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177421.519061:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177421.519063:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177421.519064:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177421.519067:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177421.519070:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12095 00000400:00000200:3.0:1786177421.519073:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177421.519076:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177421.519078:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177421.519081:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519083:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177421.519085:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519087:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177421.519089:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177421.519091:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177421.519092:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177421.519094:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177421.519096:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177421.519098:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177421.519100:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519102:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177421.519104:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519105:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177421.519108:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177421.519110:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519112:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177421.519114:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177421.519118:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177421.519120:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519123:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177421.519126:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177421.519129:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519131:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177421.519134:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327800. 00000020:00000040:3.0:1786177421.519137:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177421.519139:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177421.519140:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519142:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177421.519144:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177421.519146:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177421.519150:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9400. 00000100:00000001:3.0:1786177421.519153:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177421.519158:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177421.519175:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177421.519178:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177421.519182:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000038e8ffa8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177421.519189:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177421.519191:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177421.519194:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177421.519195:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177421.519197:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177421.519199:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177421.519201:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12095 00000400:00000200:3.0:1786177421.519204:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177421.519207:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177421.519209:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177421.519320:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519322:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177421.519324:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519326:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177421.519329:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177421.519330:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177421.519331:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177421.519333:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177421.519335:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177421.519337:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177421.519340:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519342:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177421.519343:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519346:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177421.519348:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177421.519350:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519352:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177421.519354:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177421.519395:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177421.519398:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519400:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177421.519402:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177421.519404:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519405:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177421.519409:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326d80. 00000020:00000040:3.0:1786177421.519411:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177421.519413:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177421.519414:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519416:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177421.519418:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177421.519419:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177421.519423:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9000. 00000100:00000001:3.0:1786177421.519426:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177421.519430:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177421.519436:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177421.519438:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177421.519442:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000adba13d3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177421.519448:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177421.519451:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177421.519453:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177421.519455:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177421.519456:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177421.519458:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177421.519461:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12095 00000400:00000200:3.0:1786177421.519465:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177421.519467:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177421.519469:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177421.519471:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519473:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177421.519475:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519477:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177421.519479:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177421.519480:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177421.519481:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177421.519483:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177421.519485:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177421.519487:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177421.519489:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519491:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177421.519493:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519494:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177421.519496:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177421.519499:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519501:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177421.519503:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177421.519505:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177421.519519:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519522:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177421.519525:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177421.519527:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519529:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177421.519532:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325180. 00000020:00000040:3.0:1786177421.519535:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177421.519537:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177421.519538:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519539:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177421.519540:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177421.519542:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177421.519545:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa800. 00000100:00000001:3.0:1786177421.519548:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177421.519553:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177421.519560:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177421.519562:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177421.519565:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051326aff] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177421.519584:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177421.519586:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177421.519589:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177421.519590:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177421.519592:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177421.519594:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177421.519597:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12095 00000400:00000200:3.0:1786177421.519601:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177421.519604:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177421.519606:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177421.519609:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519611:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177421.519612:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519615:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177421.519617:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177421.519618:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177421.519619:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177421.519621:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177421.519623:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177421.519625:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177421.519627:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519629:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177421.519630:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519632:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177421.519634:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177421.519637:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177421.519639:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177421.519641:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177421.519644:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177421.519646:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519647:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177421.519649:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177421.519652:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519653:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177421.519781:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786177421.519785:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177421.519787:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177421.519788:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177421.519790:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177421.519791:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177421.519793:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177421.519797:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786177421.519800:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177421.519805:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177421.519817:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177421.519819:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177421.519822:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177421.519830:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177421.519834:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12106) 00000100:00000001:0.0:1786177421.520790:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177421.520795:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177421.520797:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177421.520805:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.520814:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177421.520817:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177421.520820:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177421.520823:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177421.520826:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177421.520830:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235615360:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177421.520834:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177421.520837:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177421.520840:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177421.520842:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177421.520857:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177421.520860:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177421.520866:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177421.520870:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177421.520876:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177421.520883:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235615360, portal 25 00000100:00000001:0.0:1786177421.520886:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177421.520888:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280814464 : -110371428737152 : ffff9b9e26325f80) 00000100:00000040:0.0:1786177421.520893:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.520901:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177421.520903:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177421.520906:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177421.520909:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235615360, offset 0 00000400:00000200:0.0:1786177421.520915:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177421.520922:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177421.520928:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177421.520933:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177421.520937:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177421.520940:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177421.520942:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177421.520945:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426874:426874:256:4294967295] 192.168.202.132@tcp LPNI seq info [426836:426836:8:4294967295] 00000400:00000200:0.0:1786177421.520950:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177421.520958:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177421.520964:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177421.520969:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177421.520976:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177421.520980:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1786177421.520986:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177421.520989:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000100:0.0:1786177421.520994:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177421.520996:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177421.521002:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177421.521006:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177421.521010:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177421.521012:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177421.521015:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177421.521019:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.521029:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177421.521033:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.521041:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177421.521043:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177421.521048:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177421.521052:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177421.521054:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306772228 00000100:00000010:0.0:1786177421.521058:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5ae0. 00000100:00000001:0.0:1786177421.521064:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177421.521071:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306772228 00000100:00000010:0.0:1786177421.521074:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d52a0. 00000100:00000001:0.0:1786177421.521077:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177421.521080:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177421.521082:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177421.521084:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177421.521089:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177421/real 1786177421] req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177421.521097:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177421.521099:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177421.521101:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177421.521104:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177421.521105:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177421.521108:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177421.521111:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177421.521119:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.521126:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177421.521127:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.521130:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177421.521131:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177421.521135:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.521142:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177421.521147:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.521154:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177421.521157:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.521158:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177421.521160:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.521162:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177421.521164:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177421.521179:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177421.521183:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177421.521186:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177421.521191:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177421.521197:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235615360:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177421.521203:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177421.521206:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.521325:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177421.521327:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177421.521329:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177421.521331:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177421.521333:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177421.521337:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615488:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177421.521340:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177421.521342:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177421.521344:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177421.521346:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177421.521349:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177421.521351:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177421.521354:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177421.521356:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177421.521389:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177421.521393:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235615488, portal 10 00000100:00000001:0.0:1786177421.521394:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177421.521396:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280820736 : -110371428730880 : ffff9b9e26327800) 00000100:00000040:0.0:1786177421.521402:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.521408:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177421.521410:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177421.521412:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177421.521414:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235615488, offset 0 00000400:00000200:0.0:1786177421.521417:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177421.521421:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177421.521425:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177421.521428:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177421.521430:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177421.521432:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177421.521434:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177421.521436:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426875:426875:256:4294967295] 192.168.202.132@tcp LPNI seq info [426837:426837:8:4294967295] 00000400:00000200:0.0:1786177421.521440:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177421.521446:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177421.521451:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177421.521454:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177421.521458:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177421.521461:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1786177421.521465:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177421.521468:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000100:0.0:1786177421.521472:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177421.521473:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177421.521477:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177421.521480:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177421.521483:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177421.521484:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177421.521486:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177421.521489:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.521497:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177421.521500:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.521506:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177421.522933:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177421.522947:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177421.522953:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177421.522959:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177421.522962:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306768393 00000100:00000001:0.0:1786177421.522966:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306772228 00000100:00000001:0.0:1786177421.522969:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306768393 00000100:00000001:0.0:1786177421.522971:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306772228 00000100:00000001:0.0:1786177421.522973:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177421.522976:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177421.522979:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177421.522987:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177421/real 1786177421] req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177421.522997:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177421.522999:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177421.523002:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177421.523004:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177421.523007:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177421.523010:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177421.523013:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177421.523021:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.523029:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177421.523030:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.523032:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177421.523034:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177421.523037:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.523045:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177421.523050:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.523057:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177421.523060:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.523062:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177421.523063:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.523065:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177421.523068:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177421.523071:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177421.523075:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177421.523077:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177421.523082:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177421.523090:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615488:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177421.523094:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177421.523098:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.523104:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177421.523106:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177421.523108:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177421.523111:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177421.523114:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177421.523117:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615616:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177421.523120:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177421.523124:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177421.523126:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177421.523128:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177421.523134:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177421.523136:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177421.523140:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177421.523142:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177421.523146:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177421.523150:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235615616, portal 10 00000100:00000001:0.0:1786177421.523152:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177421.523153:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280818048 : -110371428733568 : ffff9b9e26326d80) 00000100:00000040:0.0:1786177421.523160:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.523178:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177421.523180:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177421.523183:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177421.523186:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235615616, offset 0 00000400:00000200:0.0:1786177421.523191:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177421.523196:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177421.523203:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177421.523206:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177421.523209:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177421.524308:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177421.524312:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177421.524316:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426876:426876:256:4294967295] 192.168.202.132@tcp LPNI seq info [426838:426838:8:4294967295] 00000400:00000200:0.0:1786177421.524321:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177421.524331:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177421.524336:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177421.524342:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177421.524348:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177421.524351:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1786177421.524369:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177421.524373:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000100:0.0:1786177421.524377:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177421.524380:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177421.524384:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177421.524389:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177421.524393:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177421.524395:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177421.524398:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177421.524403:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.524414:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177421.524416:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.524429:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177421.524430:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177421.524435:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177421.524438:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177421.524442:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177421.524445:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306768395 00000100:00000001:0.0:1786177421.524447:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306772228 00000100:00000001:0.0:1786177421.524450:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306768395 00000100:00000001:0.0:1786177421.524452:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306772228 00000100:00000001:0.0:1786177421.524454:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177421.524456:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177421.524459:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177421.524463:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177421/real 1786177421] req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177421.524471:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177421.524472:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177421.524474:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177421.524476:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177421.524478:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177421.524480:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177421.524484:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177421.524491:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.524498:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177421.524499:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.524501:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177421.524503:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177421.524506:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.525628:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177421.525636:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.525643:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177421.525647:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.525649:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177421.525650:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.525653:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177421.525656:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177421.525660:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177421.525664:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177421.525667:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177421.525672:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177421.525680:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615616:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177421.525686:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177421.525690:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.525696:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177421.525698:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177421.525701:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177421.525704:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177421.525706:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177421.525709:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615744:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177421.525713:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177421.525716:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177421.525719:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177421.525721:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177421.525738:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177421.525740:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177421.525745:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177421.525749:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177421.525754:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177421.525759:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235615744, portal 4 00000100:00000001:0.0:1786177421.525761:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177421.525762:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280810880 : -110371428740736 : ffff9b9e26325180) 00000100:00000040:0.0:1786177421.525769:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.525775:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177421.525778:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177421.525780:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177421.525783:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235615744, offset 0 00000400:00000200:0.0:1786177421.525788:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177421.525795:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177421.525801:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177421.525805:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177421.525808:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177421.525811:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177421.525813:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177421.525816:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426877:426877:256:4294967295] 192.168.202.132@tcp LPNI seq info [426839:426839:8:4294967295] 00000400:00000200:0.0:1786177421.525820:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177421.525827:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177421.525832:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177421.525837:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177421.525842:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177421.525844:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1786177421.528812:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177421.528819:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000100:0.0:1786177421.528824:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177421.528828:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177421.528836:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177421.528841:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177421.529954:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177421.529959:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177421.529966:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177421.529975:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.529988:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177421.529992:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.529999:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177421.530001:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177421.530008:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177421.530014:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177421.530019:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177421.530022:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306768400 00000100:00000001:0.0:1786177421.530026:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306772228 00000100:00000001:0.0:1786177421.530028:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306768400 00000100:00000001:0.0:1786177421.530030:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306772228 00000100:00000001:0.0:1786177421.530032:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177421.530035:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177421.530038:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177421.530043:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177421/real 1786177421] req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177421.530053:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177421.530055:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177421.530057:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177421.530061:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177421.530063:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177421.530066:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177421.530069:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177421.530088:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.530096:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177421.530098:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.530101:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177421.530103:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177421.530107:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.530113:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177421.530117:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.530124:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177421.530127:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.530128:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177421.530129:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.530132:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177421.530134:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177421.530137:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177421.530141:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177421.530144:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177421.530147:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177421.530155:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615744:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177421.530160:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177421.530164:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.530181:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177421.530183:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177421.530186:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177421.530189:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177421.530191:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177421.530195:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615872:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177421.530198:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177421.530201:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177421.530204:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177421.530206:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177421.531748:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177421.531756:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177421.531764:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177421.531771:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177421.531777:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177421.531784:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235615872, portal 4 00000100:00000001:0.0:1786177421.531786:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177421.531788:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280813568 : -110371428738048 : ffff9b9e26325c00) 00000100:00000040:0.0:1786177421.531798:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.531806:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177421.531809:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177421.531811:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177421.531814:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235615872, offset 0 00000400:00000200:0.0:1786177421.531819:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177421.531826:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177421.531833:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177421.531838:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177421.531842:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177421.532442:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177421.532446:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177421.532450:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426878:426878:256:4294967295] 192.168.202.132@tcp LPNI seq info [426840:426840:8:4294967295] 00000400:00000200:0.0:1786177421.532455:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177421.532465:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177421.532472:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177421.532477:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177421.532484:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177421.532487:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1786177421.532492:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177421.532496:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 11 secs later) 00000800:00000100:0.0:1786177421.532500:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177421.532503:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177421.532789:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177421.532795:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177421.532801:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177421.532803:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177421.532808:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177421.532813:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.532825:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177421.532828:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177446 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.532836:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177421.532837:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177421.532843:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177421.532847:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177421.532860:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177421.532863:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306768405 00000100:00000001:0.0:1786177421.532866:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306772228 00000100:00000001:0.0:1786177421.532869:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306768405 00000100:00000001:0.0:1786177421.532871:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306772228 00000100:00000001:0.0:1786177421.532873:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177421.532877:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177421.532880:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177421.532884:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177421/real 1786177421] req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177421.532893:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177421.532895:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177421.532897:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177421.532900:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177421.532902:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177421.532904:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177421.532907:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177421.532915:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.532922:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177421.532924:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.532926:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177421.532928:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177421.532931:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.532937:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177421.532942:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.532949:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177421.532952:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.532953:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177421.532954:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.532957:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177421.532959:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177421.532962:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177421.532967:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177421.532970:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177421.532974:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177421.532982:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235615872:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177421.532986:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.532989:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177421.532993:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325f80 x1872930235615360/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.532999:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177421.533002:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177421.533004:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177421.533007:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177421.533010:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177421.533012:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177421.533014:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177421.533017:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa000. 02000000:00000001:0.0:1786177421.533020:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177421.533022:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177421.533024:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325f80. 00000100:00000001:0.0:1786177421.533027:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177421.533029:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.533030:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177421.533034:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327800 x1872930235615488/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.533042:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177421.533044:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177421.533045:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177421.533048:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177421.533049:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177421.533050:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177421.533052:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177421.533053:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9400. 02000000:00000001:0.0:1786177421.533055:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177421.533056:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177421.533057:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327800. 00000100:00000001:0.0:1786177421.533060:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177421.533061:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.533062:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177421.533066:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326d80 x1872930235615616/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.533072:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177421.533073:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177421.533074:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177421.534949:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177421.534954:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177421.534956:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177421.534959:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177421.534963:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9000. 02000000:00000001:0.0:1786177421.534967:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177421.534969:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177421.534971:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326d80. 00000100:00000001:0.0:1786177421.534975:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177421.534977:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.534980:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177421.534987:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325180 x1872930235615744/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.534997:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177421.534999:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177421.535001:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177421.535003:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177421.535004:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177421.535005:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177421.535007:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177421.535008:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa800. 02000000:00000001:0.0:1786177421.535010:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177421.535011:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177421.535012:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325180. 00000100:00000001:0.0:1786177421.535015:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177421.535016:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.535018:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177421.535021:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325c00 x1872930235615872/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177446 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177421.535028:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177421.535030:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177421.535031:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1c00. 02000000:00000001:0.0:1786177421.535033:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177421.535034:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177421.535035:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177421.535037:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177421.535038:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 02000000:00000001:0.0:1786177421.535040:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177421.535041:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177421.535043:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325c00. 00000100:00000001:0.0:1786177421.535045:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177421.535046:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.535049:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177421.535056:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177421.535059:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177421.535061:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177421.535063:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177421.772582:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177421.772589:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177421.772591:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177421.772596:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177421.772600:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177421.772604:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177421.772607:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177421.772609:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177421.772612:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177422.221275:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177422.221282:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177422.221284:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177422.221289:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177422.221293:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177422.221298:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177422.221300:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177422.221303:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177422.221306:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177422.413315:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177422.413326:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177422.413334:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177422.413338:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177422.413341:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177422.413345:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177422.413349:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177422.413351:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177422.413353:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177422.413374:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177422.413380:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177422.413381:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177422.413473:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177422.413479:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177422.413483:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177422.796687:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177422.796694:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177422.796696:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177422.796702:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177422.796705:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177422.796709:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177422.796712:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177422.796714:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177422.796717:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177423.245141:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177423.245175:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177423.245177:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177423.245183:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177423.245186:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177423.245189:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177423.245192:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177423.245194:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177423.245197:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177423.436477:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177423.436485:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177423.436488:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177423.436491:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177423.436495:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177423.436498:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177423.436500:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177423.436502:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177423.436506:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177423.436508:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177423.436511:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177423.436513:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177423.436514:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177423.436515:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177423.436517:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177423.436518:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177423.436519:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177423.436521:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177423.436621:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177423.436624:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177423.436627:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177423.820585:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177423.820592:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177423.820594:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177423.820599:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177423.820603:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177423.820606:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177423.820609:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177423.820612:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177423.820615:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177424.268662:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177424.268670:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177424.268672:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177424.268677:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177424.268682:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177424.268686:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177424.268688:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177424.268691:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177424.268694:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177424.460480:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177424.460488:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177424.460494:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177424.460495:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177424.460498:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177424.460499:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177424.460616:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177424.460620:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177424.460623:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177424.844993:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177424.845001:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177424.845004:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177424.845009:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177424.845013:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177424.845017:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177424.845019:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177424.845022:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177424.845025:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177425.292581:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177425.292649:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177425.292651:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177425.292932:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177425.292937:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177425.292942:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177425.292944:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177425.292947:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177425.292950:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177425.423616:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306772228, at 4306772291 00000100:00000010:1.0:1786177425.423630:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d52a0. 00000100:00000040:0.0:1786177425.423711:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306772228, at 4306772291 00000100:00000010:0.0:1786177425.423730:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5ae0. 00010000:00000001:0.0:1786177425.484558:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177425.484566:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177425.484573:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177425.484574:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177425.484577:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177425.484578:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177425.484690:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177425.484694:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177425.484697:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177425.869160:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177425.869168:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177425.869170:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177425.869175:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177425.869179:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177425.869182:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177425.869185:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177425.869188:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177425.869191:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177426.317078:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177426.317086:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177426.317088:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177426.317094:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177426.317099:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177426.317104:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177426.317107:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177426.317110:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177426.317113:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177426.508742:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177426.508752:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177426.508759:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177426.508761:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177426.508764:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177426.508766:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177426.508877:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177426.508881:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177426.508885:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177426.639759:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177426.639774:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177426.639777:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177426.639782:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177426.639788:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177426.639798:0:191280:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12101 00000400:00000200:3.0:1786177426.639810:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177426.639817:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177426.639821:0:191280:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177426.639824:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.639828:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177426.639831:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.639834:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177426.639839:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177426.639841:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177426.639844:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177426.639848:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177426.639851:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177426.639855:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177426.639858:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.639860:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177426.639861:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.639863:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177426.639866:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177426.639870:0:191280:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177426.639873:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177426.639877:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177426.639881:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177426.639884:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.639886:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177426.639889:0:191280:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177426.639903:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786177426.639907:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177426.639911:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177426.639913:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.639917:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177426.639921:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177426.639926:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177426.639937:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786177426.639946:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177426.639958:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177426.639969:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177426.639975:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177426.639980:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177426.640011:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177426.640014:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177426.640018:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177426.640020:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177426.640022:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177426.640024:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177426.640027:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12101 00000400:00000200:3.0:1786177426.640030:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177426.640033:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177426.640035:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177426.640038:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640040:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177426.640042:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640044:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177426.640046:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177426.640047:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177426.640049:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177426.640051:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177426.640053:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177426.640055:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177426.640059:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640061:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177426.640062:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640064:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177426.640066:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177426.640068:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640070:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177426.640072:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177426.640079:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177426.640081:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640083:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177426.640089:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177426.640091:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640094:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177426.640097:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786177426.640099:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177426.640102:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177426.640103:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640104:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177426.640105:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177426.640107:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177426.640111:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786177426.640115:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177426.640130:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177426.640137:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177426.640139:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177426.640142:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177426.640150:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177426.640152:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177426.640156:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177426.640157:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177426.640159:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177426.640161:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177426.640164:0:191280:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12101 00000400:00000200:3.0:1786177426.640168:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177426.640170:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177426.640172:0:191280:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177426.640175:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640177:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177426.640179:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640181:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177426.640183:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177426.640185:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177426.640187:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177426.640190:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177426.640192:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177426.640194:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177426.640197:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640198:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177426.640200:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640202:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177426.640205:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177426.640207:0:191280:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640209:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177426.640211:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177426.640213:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177426.640216:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640218:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177426.640220:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177426.640222:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640224:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177426.640227:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786177426.640229:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177426.640232:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177426.640233:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640235:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177426.640236:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177426.640237:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177426.640241:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177426.640244:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177426.640248:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177426.640255:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177426.640269:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177426.640275:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177426.640282:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177426.640285:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177426.640288:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177426.640290:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177426.640292:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177426.640295:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177426.640297:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12101 00000400:00000200:3.0:1786177426.640300:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177426.640303:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177426.640305:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177426.640308:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640310:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177426.640312:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640314:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177426.640316:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177426.640317:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177426.640319:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177426.640321:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177426.640323:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177426.640325:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177426.640328:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640330:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177426.640331:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640333:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177426.640335:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177426.640337:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640339:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177426.640341:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177426.640345:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177426.640347:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640349:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177426.640351:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177426.640354:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640381:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177426.640385:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786177426.640387:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177426.640390:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177426.640391:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640393:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177426.640395:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177426.640396:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177426.640400:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786177426.640403:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177426.640407:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177426.640414:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177426.640416:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177426.640420:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177426.640427:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177426.640430:0:191280:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177426.640433:0:191280:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177426.640435:0:191280:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177426.640436:0:191280:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177426.640439:0:191280:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177426.640442:0:191280:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12101 00000400:00000200:3.0:1786177426.640445:0:191280:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177426.640448:0:191280:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177426.640450:0:191280:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177426.640453:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640455:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177426.640457:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640459:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177426.640461:0:191280:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177426.640462:0:191280:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177426.640464:0:191280:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177426.640466:0:191280:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177426.640467:0:191280:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177426.640470:0:191280:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177426.640472:0:191280:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640474:0:191280:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177426.640476:0:191280:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640478:0:191280:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177426.640480:0:191280:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177426.640482:0:191280:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177426.640484:0:191280:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177426.640486:0:191280:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177426.640488:0:191280:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177426.640490:0:191280:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640491:0:191280:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177426.640493:0:191280:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177426.640495:0:191280:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640497:0:191280:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177426.640500:0:191280:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786177426.640502:0:191280:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177426.640505:0:191280:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177426.640506:0:191280:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177426.640508:0:191280:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177426.640509:0:191280:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177426.640510:0:191280:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177426.640514:0:191280:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786177426.640518:0:191280:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177426.640522:0:191280:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177426.640528:0:191280:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177426.640530:0:191280:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177426.640534:0:191280:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177426.640553:0:191280:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177426.640558:0:191280:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12111) 00000100:00000001:0.0:1786177426.640841:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177426.640847:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177426.640849:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177426.640857:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.640867:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177426.640869:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177426.640873:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177426.640876:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177426.640879:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177426.640882:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235616000:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177426.640887:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177426.640892:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177426.640895:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177426.640898:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177426.640904:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177426.640906:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177426.640912:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177426.640917:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177426.640926:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177426.640935:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235616000, portal 25 00000100:00000001:0.0:1786177426.640938:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177426.640940:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280815360 : -110371428736256 : ffff9b9e26326300) 00000100:00000040:0.0:1786177426.640945:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.640952:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177426.640955:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177426.640958:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177426.640962:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235616000, offset 0 00000400:00000200:0.0:1786177426.640970:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177426.640980:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177426.640988:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177426.640994:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177426.640999:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177426.641005:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177426.641008:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177426.641012:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426879:426879:256:4294967295] 192.168.202.132@tcp LPNI seq info [426841:426841:8:4294967295] 00000400:00000200:0.0:1786177426.641016:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177426.641026:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177426.641033:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177426.641039:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177426.641047:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177426.641052:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1786177426.641057:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177426.641062:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:0.0:1786177426.641066:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177426.641069:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177426.641076:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177426.641081:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177426.641087:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177426.641089:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177426.641094:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177426.641098:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641108:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177426.641111:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641129:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177426.641132:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177426.641140:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177426.641144:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177426.641147:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306777348 00000100:00000010:0.0:1786177426.641153:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5ae0. 00000100:00000001:0.0:1786177426.641161:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177426.641170:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306777348 00000100:00000010:0.0:1786177426.641174:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d58a0. 00000100:00000001:0.0:1786177426.641178:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177426.641181:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177426.641184:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177426.641187:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177426.641191:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177426/real 1786177426] req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177426.641201:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177426.641203:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177426.641205:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177426.641208:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177426.641210:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177426.641212:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177426.641216:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177426.641224:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641231:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177426.641233:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.641236:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177426.641239:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177426.641243:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641250:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177426.641254:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641272:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177426.641275:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.641277:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177426.641278:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.641281:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177426.641283:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177426.641287:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177426.641291:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177426.641295:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177426.641299:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177426.641306:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235616000:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177426.641312:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177426.641315:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641322:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177426.641323:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177426.641325:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177426.641327:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177426.641329:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177426.641332:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616128:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177426.641336:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177426.641338:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177426.641340:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177426.641342:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177426.641345:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177426.641347:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177426.641350:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177426.641353:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177426.641378:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177426.641382:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235616128, portal 10 00000100:00000001:0.0:1786177426.641384:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177426.641386:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280821632 : -110371428729984 : ffff9b9e26327b80) 00000100:00000040:0.0:1786177426.641393:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641406:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177426.641408:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177426.641410:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177426.641413:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235616128, offset 0 00000400:00000200:0.0:1786177426.641416:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177426.641421:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177426.641424:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177426.641427:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177426.641430:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177426.641432:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177426.641434:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177426.641438:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426880:426880:256:4294967295] 192.168.202.132@tcp LPNI seq info [426842:426842:8:4294967295] 00000400:00000200:0.0:1786177426.641442:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177426.641449:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177426.641453:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177426.641456:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177426.641460:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177426.641463:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1786177426.641467:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177426.641470:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:0.0:1786177426.641474:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177426.641476:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177426.641479:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177426.641482:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177426.641485:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177426.641487:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177426.641490:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177426.641493:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641501:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177426.641504:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641516:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177426.641518:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177426.641521:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177426.641525:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177426.641528:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177426.641530:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306773512 00000100:00000001:0.0:1786177426.641532:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306777348 00000100:00000001:0.0:1786177426.641535:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306773512 00000100:00000001:0.0:1786177426.641537:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306777348 00000100:00000001:0.0:1786177426.641550:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177426.641552:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177426.641555:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177426.641558:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177426/real 1786177426] req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177426.641566:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177426.641568:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177426.641570:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177426.641572:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177426.641574:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177426.641576:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177426.641579:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177426.641588:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641592:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177426.641594:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.641595:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177426.641597:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177426.641600:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641606:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177426.641610:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641616:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177426.641618:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.641620:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177426.641621:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.641622:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177426.641625:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177426.641628:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177426.641631:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177426.641634:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177426.641638:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177426.641646:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616128:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177426.641650:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177426.641654:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641661:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177426.641662:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177426.641664:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177426.641666:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177426.641668:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177426.641671:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616256:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177426.641674:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177426.641677:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177426.641679:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177426.641681:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177426.641684:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177426.641686:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177426.641689:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177426.641691:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177426.641694:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177426.641697:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235616256, portal 10 00000100:00000001:0.0:1786177426.641699:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177426.641700:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280812672 : -110371428738944 : ffff9b9e26325880) 00000100:00000040:0.0:1786177426.641706:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641713:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177426.641715:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177426.641717:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177426.641719:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235616256, offset 0 00000400:00000200:0.0:1786177426.641723:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177426.641727:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177426.641730:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177426.641732:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177426.641735:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177426.641737:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177426.641739:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177426.641742:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426881:426881:256:4294967295] 192.168.202.132@tcp LPNI seq info [426843:426843:8:4294967295] 00000400:00000200:0.0:1786177426.641747:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177426.641753:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177426.641757:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177426.641759:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177426.641763:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177426.641766:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1786177426.641771:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177426.641774:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:0.0:1786177426.641777:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177426.641780:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177426.641782:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177426.641786:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177426.641789:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177426.641790:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177426.641793:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177426.641796:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641804:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177426.641807:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641813:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177426.641815:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177426.641817:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177426.641820:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177426.641823:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177426.641825:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306773512 00000100:00000001:0.0:1786177426.641828:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306777348 00000100:00000001:0.0:1786177426.641830:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306773512 00000100:00000001:0.0:1786177426.641832:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306777348 00000100:00000001:0.0:1786177426.641834:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177426.641836:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177426.641837:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177426.641841:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177426/real 1786177426] req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177426.641848:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177426.641849:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177426.641851:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177426.641853:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177426.641855:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177426.641857:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177426.641860:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177426.641869:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641875:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177426.641876:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.641877:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177426.641879:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177426.641882:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641888:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177426.641892:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641898:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177426.641901:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.641902:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177426.641904:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.641906:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177426.641908:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177426.641910:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177426.641913:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177426.641915:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177426.641919:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177426.641925:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616256:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177426.641928:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177426.641933:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641941:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177426.641942:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177426.641944:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177426.641946:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177426.641948:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177426.641950:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616384:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177426.641954:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177426.641956:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177426.641957:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177426.641959:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177426.641962:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177426.641964:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177426.641967:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177426.641969:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177426.641972:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177426.641975:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235616384, portal 4 00000100:00000001:0.0:1786177426.641977:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177426.641978:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280811776 : -110371428739840 : ffff9b9e26325500) 00000100:00000040:0.0:1786177426.641984:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.641991:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177426.641993:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177426.641995:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177426.641997:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235616384, offset 0 00000400:00000200:0.0:1786177426.642001:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177426.642004:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177426.642007:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177426.642009:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177426.642012:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177426.642014:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177426.642017:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177426.642020:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426882:426882:256:4294967295] 192.168.202.132@tcp LPNI seq info [426844:426844:8:4294967295] 00000400:00000200:0.0:1786177426.642025:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177426.642031:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177426.642035:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177426.642038:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177426.642041:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177426.642044:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1786177426.642048:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177426.642051:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:0.0:1786177426.642055:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177426.642056:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177426.642059:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177426.642063:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177426.642065:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177426.642067:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177426.642069:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177426.642072:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642080:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177426.642083:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642090:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177426.642091:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177426.642094:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177426.642096:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177426.642099:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177426.642101:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306773512 00000100:00000001:0.0:1786177426.642103:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306777348 00000100:00000001:0.0:1786177426.642105:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306773512 00000100:00000001:0.0:1786177426.642107:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306777348 00000100:00000001:0.0:1786177426.642109:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177426.642112:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177426.642114:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177426.642127:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177426/real 1786177426] req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177426.642135:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177426.642136:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177426.642138:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177426.642141:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177426.642143:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177426.642144:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177426.642147:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177426.642155:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642161:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177426.642163:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642165:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177426.642166:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177426.642170:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642176:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177426.642180:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642186:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177426.642188:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642190:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177426.642191:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642193:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177426.642194:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177426.642196:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177426.642201:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177426.642202:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177426.642207:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177426.642215:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616384:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177426.642219:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177426.642223:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642229:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177426.642231:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177426.642233:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177426.642235:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177426.642237:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177426.642239:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616512:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177426.642242:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177426.642244:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177426.642245:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177426.642247:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177426.642251:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177426.642253:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177426.642256:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177426.642258:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177426.642261:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177426.642264:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235616512, portal 4 00000100:00000001:0.0:1786177426.642266:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177426.642268:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280806400 : -110371428745216 : ffff9b9e26324000) 00000100:00000040:0.0:1786177426.642274:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642280:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177426.642284:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177426.642285:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177426.642288:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235616512, offset 0 00000400:00000200:0.0:1786177426.642291:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177426.642295:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177426.642298:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177426.642301:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177426.642303:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177426.642305:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177426.642307:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177426.642310:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426883:426883:256:4294967295] 192.168.202.132@tcp LPNI seq info [426845:426845:8:4294967295] 00000400:00000200:0.0:1786177426.642314:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177426.642320:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177426.642324:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177426.642327:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177426.642331:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177426.642334:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1786177426.642338:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177426.642341:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 6 secs later) 00000800:00000100:0.0:1786177426.642344:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177426.642346:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177426.642349:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177426.642353:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177426.642356:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177426.642378:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177426.642381:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177426.642385:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642393:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177426.642396:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177451 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642402:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177426.642404:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177426.642407:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177426.642410:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177426.642413:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177426.642415:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306773515 00000100:00000001:0.0:1786177426.642418:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306777348 00000100:00000001:0.0:1786177426.642420:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306773515 00000100:00000001:0.0:1786177426.642422:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306777348 00000100:00000001:0.0:1786177426.642424:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177426.642427:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177426.642428:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177426.642432:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177426/real 1786177426] req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177426.642439:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177426.642440:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177426.642442:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177426.642444:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177426.642446:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177426.642448:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177426.642451:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177426.642459:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642465:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177426.642466:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642467:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177426.642469:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177426.642472:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642478:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177426.642481:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642489:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177426.642491:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642493:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177426.642494:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642496:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177426.642497:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177426.642500:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177426.642503:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177426.642504:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177426.642508:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177426.642515:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616512:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177426.642520:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642522:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177426.642525:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326300 x1872930235616000/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642533:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177426.642536:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177426.642539:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177426.642542:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177426.642544:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177426.642546:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177426.642548:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177426.642551:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:0.0:1786177426.642556:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177426.642558:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177426.642560:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326300. 00000100:00000001:0.0:1786177426.642563:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177426.642564:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642567:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177426.642570:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327b80 x1872930235616128/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642576:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177426.642578:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177426.642579:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177426.642581:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177426.642582:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177426.642583:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177426.642585:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177426.642586:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:0.0:1786177426.642588:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177426.642589:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177426.642590:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327b80. 00000100:00000001:0.0:1786177426.642592:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177426.642593:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642595:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177426.642598:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325880 x1872930235616256/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642603:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177426.642604:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177426.642605:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177426.642607:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177426.642608:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177426.642609:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177426.642610:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177426.642611:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:0.0:1786177426.642613:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177426.642614:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177426.642615:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325880. 00000100:00000001:0.0:1786177426.642617:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177426.642618:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642619:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177426.642623:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325500 x1872930235616384/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642636:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177426.642637:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177426.642639:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3400. 02000000:00000001:0.0:1786177426.642641:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177426.642642:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177426.642643:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177426.642644:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177426.642646:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:0.0:1786177426.642647:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177426.642648:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177426.642650:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325500. 00000100:00000001:0.0:1786177426.642652:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177426.642653:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642654:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177426.642657:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324000 x1872930235616512/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177451 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177426.642662:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177426.642663:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177426.642665:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0c00. 02000000:00000001:0.0:1786177426.642666:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177426.642667:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177426.642668:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177426.642670:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177426.642671:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8400. 02000000:00000001:0.0:1786177426.642673:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177426.642674:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177426.642675:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324000. 00000100:00000001:0.0:1786177426.642678:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177426.642679:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642681:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177426.642690:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177426.642692:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177426.642695:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177426.642697:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177426.892659:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177426.892672:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177426.892674:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177426.892683:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177426.892688:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177426.892692:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177426.892695:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177426.892699:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177426.892703:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177427.340533:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177427.340543:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177427.340546:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177427.340553:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177427.340558:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177427.340563:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177427.340566:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177427.340569:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177427.340574:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177427.532513:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177427.532522:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177427.532529:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177427.532531:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177427.532534:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177427.532535:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177427.532638:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177427.532643:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177427.532646:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177427.916534:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177427.916545:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177427.916548:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177427.916555:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177427.916560:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177427.916565:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177427.916569:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177427.916572:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177427.916576:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1786177428.236880:0:2384:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e101ff9c0] -> 12345-192.168.202.32@tcp (6) 00000800:00000200:3.0:1786177428.236890:0:2384:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.32@tcp ip 192.168.202.32:988 00000800:00000200:3.0:1786177428.236896:0:2384:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9b9e06819600 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:0.0:1786177428.237124:0:2385:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000001:0.0:1786177428.237131:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786177428.237146:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1786177428.237152:0:2385:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:0.0:1786177428.237155:0:2385:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177428.364517:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177428.364527:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177428.364529:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177428.364537:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177428.364542:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177428.364546:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177428.364550:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177428.364553:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177428.364557:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177428.556513:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177428.556522:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177428.556529:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177428.556530:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177428.556534:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177428.556535:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177428.556610:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177428.556615:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177428.556618:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177428.940863:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177428.940874:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177428.940877:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177428.940883:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177428.940888:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177428.940892:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177428.940895:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177428.940898:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177428.940901:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177429.388495:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177429.388503:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177429.388506:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177429.388512:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177429.388517:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177429.388521:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177429.388524:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177429.388527:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177429.388531:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177429.580610:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177429.580619:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177429.580628:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177429.580629:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177429.580633:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177429.580634:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177429.581917:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177429.581923:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177429.581929:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177429.964862:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177429.964873:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177429.964887:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177429.964894:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177429.964898:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177429.964904:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177429.964907:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177429.964910:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177429.964914:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177430.412545:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177430.412556:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177430.412559:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177430.412566:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177430.412571:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177430.412575:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177430.412579:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177430.412582:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177430.412594:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177430.540693:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306777348, at 4306777408 00000100:00000010:0.0:1786177430.540713:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5ae0. 00000100:00000040:1.0:1786177430.540871:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306777348, at 4306777408 00000100:00000010:1.0:1786177430.540877:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d58a0. 00010000:00000001:0.0:1786177430.609863:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177430.609873:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177430.609881:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177430.609883:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177430.609887:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177430.609888:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177430.610142:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177430.610147:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177430.610151:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177430.988558:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177430.988571:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177430.988573:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177430.988581:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177430.988586:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177430.988592:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177430.988596:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177430.988599:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177430.988603:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177431.436764:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177431.436773:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177431.436776:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177431.436783:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177431.436787:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177431.436792:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177431.436795:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177431.436798:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177431.436802:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177431.632000:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177431.632011:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177431.632018:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177431.632022:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177431.632025:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177431.632030:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177431.632034:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177431.632036:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177431.632040:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177431.632042:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177431.632045:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177431.632047:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177431.633022:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177431.633029:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177431.633033:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177431.756637:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177431.756653:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177431.756656:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177431.756670:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177431.756676:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177431.756686:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12106 00000400:00000200:3.0:1786177431.756699:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177431.756708:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177431.756712:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177431.756716:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.756721:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177431.756724:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.756727:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177431.756732:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177431.756735:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177431.756738:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177431.756743:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177431.756746:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177431.756748:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177431.756752:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.756764:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177431.756767:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.756769:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177431.756772:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177431.756776:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177431.756779:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177431.756783:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177431.756787:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177431.756790:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.756793:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177431.756796:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177431.756809:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324a80. 00000020:00000040:3.0:1786177431.756813:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177431.756818:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177431.756821:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.756825:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177431.756828:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177431.756833:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177431.756843:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab400. 00000100:00000001:3.0:1786177431.756851:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177431.756862:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177431.756881:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177431.756886:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177431.756892:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000be8cad3] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177431.756921:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177431.756924:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177431.756928:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177431.756929:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177431.756931:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177431.756934:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177431.756937:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12106 00000400:00000200:3.0:1786177431.756940:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177431.756943:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177431.756946:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177431.756949:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.756951:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177431.756953:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.756955:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177431.756958:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177431.756959:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177431.756961:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177431.756964:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177431.756966:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177431.756968:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177431.756971:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.756972:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177431.756974:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.756976:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177431.756980:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177431.756983:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177431.756985:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177431.756987:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177431.756994:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177431.756995:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.756997:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177431.757003:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177431.757006:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.757008:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177431.757012:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324700. 00000020:00000040:3.0:1786177431.757014:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177431.757017:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177431.757018:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.757020:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177431.757021:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177431.757023:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177431.757026:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aa400. 00000100:00000001:3.0:1786177431.757030:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177431.757035:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177431.757043:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177431.757045:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177431.757049:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000adcac95] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177431.757056:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177431.757059:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177431.757062:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177431.757064:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177431.757065:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177431.757097:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177431.757101:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12106 00000400:00000200:3.0:1786177431.757104:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177431.757108:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177431.757110:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177431.757113:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.757115:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177431.757117:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.757119:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177431.757122:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177431.757131:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177431.757132:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177431.757134:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177431.757136:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177431.757138:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177431.757141:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.757143:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177431.757145:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.757147:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177431.757149:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177431.757152:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177431.757154:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177431.757156:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177431.757158:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177431.757160:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.757161:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177431.757163:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177431.757165:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.757167:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177431.757170:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326a00. 00000020:00000040:3.0:1786177431.757172:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177431.757175:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177431.757177:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.757179:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177431.757180:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177431.757182:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177431.757186:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab800. 00000100:00000001:3.0:1786177431.757189:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177431.757193:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177431.757200:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177431.757202:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177431.757206:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000091b0787f] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.758932:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177431.758943:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177431.758947:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177431.758961:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.758975:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177431.758978:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177431.758982:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177431.758986:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177431.758990:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177431.758993:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235616640:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177431.758998:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177431.759004:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177431.759007:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177431.759010:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177431.759022:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177431.759024:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177431.759032:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177431.759038:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177431.759047:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177431.759056:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235616640, portal 25 00000100:00000001:0.0:1786177431.759059:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177431.759061:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809088 : -110371428742528 : ffff9b9e26324a80) 00000100:00000040:0.0:1786177431.759078:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.759086:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177431.759097:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177431.759100:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177431.759103:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235616640, offset 0 00000400:00000200:0.0:1786177431.759112:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177431.759129:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177431.759138:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177431.759144:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177431.759148:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177431.759153:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177431.759156:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177431.759160:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426884:426884:256:4294967295] 192.168.202.132@tcp LPNI seq info [426846:426846:8:4294967295] 00000400:00000200:0.0:1786177431.759165:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177431.759176:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177431.759182:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177431.759189:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177431.759196:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177431.759201:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1786177431.759206:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:3.0:1786177431.760642:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177431.760649:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177431.760654:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177431.760656:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177431.760659:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:0.1:1786177431.760855:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1786177431.760880:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177431.760886:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177431.760897:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177431.760903:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177431.760914:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177431.760917:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177431.760924:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177431.760931:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.760946:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177431.760950:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.760958:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.760960:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177431.760971:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177431.760979:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177431.760982:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306782468 00000100:00000010:0.0:1786177431.760996:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5ae0. 00000100:00000001:0.0:1786177431.761006:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177431.761017:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306782468 00000100:00000010:0.0:1786177431.761020:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5f60. 00000100:00000001:0.0:1786177431.761024:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177431.761028:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177431.761032:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.761035:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177431.761040:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177431/real 1786177431] req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177431.761052:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177431.761054:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177431.761057:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177431.761060:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177431.761062:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177431.761064:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177431.761079:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177431.761088:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.761096:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177431.761099:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.761102:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177431.761104:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177431.761108:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.761116:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177431.761122:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.761130:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177431.761132:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.761135:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177431.761137:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.761141:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177431.761143:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177431.761147:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177431.761152:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177431.761155:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177431.761159:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177431.761169:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235616640:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177431.761176:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177431.761181:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.761190:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177431.761191:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177431.761196:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177431.761200:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177431.761203:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177431.761206:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616768:192.168.202.132@tcp:38:kworker.0 00000100:00000001:3.0:1786177431.761600:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177431.761604:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12106 00000400:00000200:3.0:1786177431.761612:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177431.761617:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177431.761620:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177431.761624:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.761637:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177431.761640:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.761642:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177431.761647:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177431.761649:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177431.761653:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177431.761657:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177431.762880:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177431.762891:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177431.762899:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.762903:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177431.762906:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.762908:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177431.762911:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177431.762915:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177431.762918:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177431.762922:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177431.762931:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177431.762934:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.762937:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177431.762943:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177431.762947:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.762949:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177431.762961:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324e00. 00000020:00000040:3.0:1786177431.762966:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177431.762972:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177431.762974:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.762978:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177431.762982:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177431.762987:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177431.762998:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8c00. 00000100:00000001:3.0:1786177431.763007:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177431.763017:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177431.763028:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177431.763034:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177431.763039:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7f0b4d2] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177431.763050:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177431.763054:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177431.763059:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177431.763061:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177431.763063:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177431.763076:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177431.763080:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12106 00000400:00000200:3.0:1786177431.763086:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177431.763092:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177431.763095:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177431.763098:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.763101:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177431.763103:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.763105:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177431.763108:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177431.763109:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177431.763111:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177431.763114:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177431.763116:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177431.763118:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177431.763121:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.763131:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177431.763133:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177431.763135:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177431.763142:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177431.763146:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177431.763148:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177431.763150:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177431.763153:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177431.763156:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.763158:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177431.763162:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177431.763164:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.763166:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177431.763172:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26324000. 00000020:00000040:3.0:1786177431.763175:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177431.763177:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177431.763178:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177431.763180:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177431.763182:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177431.763184:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177431.763188:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8400. 00000100:00000001:3.0:1786177431.763191:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177431.763195:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177431.763203:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177431.763205:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.763822:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177431.763829:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177431.763833:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177431.763836:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177431.763843:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177431.763846:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177431.764003:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177431.764009:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177431.764019:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177431.764028:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235616768, portal 10 00000100:00000001:0.0:1786177431.764030:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177431.764034:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280808192 : -110371428743424 : ffff9b9e26324700) 00000100:00000040:0.0:1786177431.764045:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.764053:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177431.764056:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177431.764059:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177431.764062:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235616768, offset 0 00000400:00000200:0.0:1786177431.764081:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177431.764098:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177431.764106:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177431.764112:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177431.764117:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177431.765602:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177431.765608:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177431.765614:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426885:426885:256:4294967295] 192.168.202.132@tcp LPNI seq info [426847:426847:8:4294967295] 00000400:00000200:0.0:1786177431.765621:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000040:3.0:1786177431.765960:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008141a9f8] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1786177431.765980:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177431.765990:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12116) 00000400:00000200:0.0:1786177431.766059:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177431.766080:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177431.766101:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177431.766111:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177431.766116:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1786177431.767255:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177431.767263:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1786177431.767270:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177431.767273:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177431.767282:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177431.767288:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177431.767297:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177431.767300:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177431.767308:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177431.767315:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.767349:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177431.767354:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.767387:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.767389:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177431.767400:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177431.767406:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177431.767412:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177431.767416:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306778638 00000100:00000001:0.0:1786177431.767420:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306782468 00000100:00000001:0.0:1786177431.767423:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306778638 00000100:00000001:0.0:1786177431.767425:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306782468 00000100:00000001:0.0:1786177431.767428:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177431.767431:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.767435:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177431.767440:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177431/real 1786177431] req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177431.767451:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177431.767453:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177431.767457:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177431.767460:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177431.767462:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177431.767465:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177431.767468:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177431.767477:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.767484:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177431.767488:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.767491:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177431.767493:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177431.767497:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.767506:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177431.767511:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.767521:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177431.767524:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.767526:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177431.767527:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.767530:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177431.767534:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177431.767538:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177431.767543:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177431.767546:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177431.767551:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177431.767560:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616768:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177431.767566:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.767570:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177431.767575:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324a80 x1872930235616640/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.767582:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177431.767587:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177431.767593:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177431.767596:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177431.767600:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177431.767603:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:0.0:1786177431.767605:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177431.767609:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab400. 02000000:00000001:0.0:1786177431.767614:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177431.767617:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177431.767619:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324a80. 00000100:00000001:0.0:1786177431.767623:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177431.767624:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.767626:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177431.767629:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324700 x1872930235616768/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.767634:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177431.767636:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177431.767637:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177431.767640:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177431.767641:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177431.767642:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177431.767644:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177431.767645:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aa400. 02000000:00000001:0.0:1786177431.767647:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177431.767649:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177431.767650:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324700. 00000100:00000001:0.0:1786177431.767652:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177431.767653:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.767656:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.767687:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177431.767689:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.767693:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177431.767696:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177431.767698:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177431.767704:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.767712:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177431.767714:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177431.767718:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177431.767722:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177431.767726:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177431.767731:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616896:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177431.767735:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177431.767742:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177431.767745:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177431.767749:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177431.767757:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177431.767759:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177431.767765:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177431.767769:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177431.767775:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177431.767782:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235616896, portal 10 00000100:00000001:0.0:1786177431.767784:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177431.767786:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280817152 : -110371428734464 : ffff9b9e26326a00) 00000100:00000040:0.0:1786177431.767794:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.767801:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177431.767803:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177431.767806:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177431.767809:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235616896, offset 0 00000400:00000200:0.0:1786177431.767815:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177431.767825:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177431.767833:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177431.767840:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177431.767844:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177431.767848:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177431.767851:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177431.767856:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426886:426886:256:4294967295] 192.168.202.132@tcp LPNI seq info [426848:426848:8:4294967295] 00000400:00000200:0.0:1786177431.767860:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177431.767869:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177431.767873:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177431.767878:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177431.767884:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177431.767887:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1786177431.767891:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177431.767894:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1786177431.767898:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177431.767900:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177431.767903:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177431.767907:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177431.767910:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177431.767911:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177431.767914:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177431.767917:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.767925:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177431.767928:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.767934:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.767935:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177431.767938:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177431.767941:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177431.767944:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177431.767946:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306778638 00000100:00000001:0.0:1786177431.767948:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306782468 00000100:00000001:0.0:1786177431.767950:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306778638 00000100:00000001:0.0:1786177431.767952:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306782468 00000100:00000001:0.0:1786177431.767954:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177431.767956:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.767958:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177431.767961:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177431/real 1786177431] req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177431.767975:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177431.767977:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177431.767978:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177431.767981:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177431.767982:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177431.767985:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177431.767988:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177431.767996:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768002:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177431.768004:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768005:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177431.768006:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177431.768010:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768017:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177431.768021:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768027:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177431.768029:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768031:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177431.768032:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768034:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177431.768036:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177431.768039:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177431.768042:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177431.768044:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177431.768048:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177431.768057:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235616896:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177431.768061:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177431.768065:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768092:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177431.768094:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177431.768097:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177431.768099:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177431.768101:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177431.768104:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617024:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177431.768107:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177431.768109:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177431.768111:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177431.768112:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177431.768116:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177431.768118:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177431.768122:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177431.768125:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177431.768128:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177431.768131:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235617024, portal 4 00000100:00000001:0.0:1786177431.768132:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177431.768134:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280809984 : -110371428741632 : ffff9b9e26324e00) 00000100:00000040:0.0:1786177431.768140:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768147:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177431.768149:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177431.768152:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177431.768154:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235617024, offset 0 00000400:00000200:0.0:1786177431.768159:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177431.768162:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177431.768166:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177431.768169:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177431.768172:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177431.768174:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177431.768176:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177431.768179:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426887:426887:256:4294967295] 192.168.202.132@tcp LPNI seq info [426849:426849:8:4294967295] 00000400:00000200:0.0:1786177431.768183:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177431.768188:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177431.768193:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177431.768196:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177431.768200:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177431.768202:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1786177431.768207:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177431.768210:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1786177431.768214:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177431.768216:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177431.768218:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177431.768221:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177431.768224:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177431.768227:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177431.768229:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177431.768232:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768240:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177431.768243:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768249:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.768250:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177431.768253:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177431.768256:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177431.768259:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177431.768262:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306778638 00000100:00000001:0.0:1786177431.768264:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306782468 00000100:00000001:0.0:1786177431.768266:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306778638 00000100:00000001:0.0:1786177431.768268:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306782468 00000100:00000001:0.0:1786177431.768271:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177431.768273:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.768275:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177431.768278:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177431/real 1786177431] req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177431.768285:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177431.768287:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177431.768289:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177431.768291:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177431.768293:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177431.768294:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177431.768297:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177431.768305:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768312:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177431.768314:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768316:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177431.768317:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177431.768320:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768327:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177431.768330:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768336:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177431.768339:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768341:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177431.768342:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768343:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177431.768345:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177431.768347:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177431.768351:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177431.768353:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177431.768385:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177431.768393:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617024:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177431.768397:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177431.768401:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768407:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177431.768409:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177431.768411:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177431.768414:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177431.768415:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177431.768418:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617152:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177431.768421:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177431.768424:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177431.768426:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177431.768428:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177431.768431:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177431.768433:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177431.768435:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177431.768438:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177431.768441:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177431.768444:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235617152, portal 4 00000100:00000001:0.0:1786177431.768446:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177431.768447:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280806400 : -110371428745216 : ffff9b9e26324000) 00000100:00000040:0.0:1786177431.768453:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768459:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177431.768462:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177431.768464:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177431.768466:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235617152, offset 0 00000400:00000200:0.0:1786177431.768470:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177431.768474:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177431.768478:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177431.768481:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177431.768483:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177431.768486:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177431.768488:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177431.768491:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426888:426888:256:4294967295] 192.168.202.132@tcp LPNI seq info [426850:426850:8:4294967295] 00000400:00000200:0.0:1786177431.768495:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177431.768501:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177431.768506:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177431.768509:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177431.768514:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.0:1786177431.768517:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1786177431.768521:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177431.768524:0:2392:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.202.132 (cnted 0, interval 60, 1 secs later) 00000800:00000100:0.0:1786177431.768528:0:2392:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177431.768530:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:0.0:1786177431.768533:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1786177431.768537:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1786177431.768540:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:0.0:1786177431.768542:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:0.0:1786177431.768544:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1786177431.768548:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768561:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177431.768564:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177456 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768570:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.768572:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1786177431.768575:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177431.768579:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177431.768582:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177431.768584:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306778641 00000100:00000001:0.0:1786177431.768587:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306782468 00000100:00000001:0.0:1786177431.768589:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306778641 00000100:00000001:0.0:1786177431.768592:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306782468 00000100:00000001:0.0:1786177431.768594:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177431.768596:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.768599:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177431.768603:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177431/real 1786177431] req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177431.768612:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177431.768613:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177431.768615:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177431.768618:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177431.768619:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177431.768622:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177431.768625:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177431.768634:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768641:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177431.768644:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768646:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177431.768648:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177431.768652:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768658:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177431.768662:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768669:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177431.768672:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768676:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177431.768678:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768680:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177431.768681:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177431.768683:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177431.768687:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177431.768690:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177431.768694:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177431.768702:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26324000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617152:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177431.768706:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768708:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177431.768712:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326a00 x1872930235616896/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768719:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177431.768721:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177431.768724:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177431.768727:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177431.768729:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177431.768731:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177431.768733:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177431.768735:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab800. 02000000:00000001:0.0:1786177431.768739:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177431.768741:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177431.768742:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326a00. 00000100:00000001:0.0:1786177431.768746:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177431.768748:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768749:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177431.768753:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324e00 x1872930235617024/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768758:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177431.768760:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177431.768761:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177431.768763:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177431.768764:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177431.768766:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177431.768768:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177431.768769:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8c00. 02000000:00000001:0.0:1786177431.768772:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177431.768773:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177431.768775:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324e00. 00000100:00000001:0.0:1786177431.768778:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177431.768780:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768782:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177431.768785:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26324000 x1872930235617152/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177456 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177431.768792:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177431.768794:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177431.768795:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177431.768797:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177431.768799:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177431.768800:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177431.768802:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177431.768803:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8400. 02000000:00000001:0.0:1786177431.768806:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177431.768808:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177431.768809:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26324000. 00000100:00000001:0.0:1786177431.768812:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177431.768813:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768817:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177431.768825:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177431.768827:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177431.768829:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177431.768831:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177432.012896:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177432.012907:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177432.012910:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177432.012917:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177432.012921:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177432.012926:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177432.012929:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177432.012934:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177432.012939:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177432.462166:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177432.462177:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177432.462181:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177432.462189:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177432.462193:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177432.462198:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177432.462202:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177432.462205:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177432.462209:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177432.653335:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177432.653346:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177432.653354:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177432.653356:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177432.653378:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177432.653379:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177432.653498:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177432.653504:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177432.653508:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177433.036768:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177433.036779:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177433.036781:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177433.036789:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177433.036793:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177433.036798:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177433.036801:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177433.036805:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177433.036809:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177433.484457:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177433.484470:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177433.484473:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177433.484481:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177433.484486:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177433.484494:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177433.484497:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177433.484500:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177433.484505:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177433.676989:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177433.677002:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177433.677005:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177433.677010:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177433.677016:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177433.677019:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177433.677022:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177433.677024:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177433.677031:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177433.677032:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177433.677036:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177433.677039:0:206639:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1786177433.677040:0:206639:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1786177433.677042:0:206639:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177433.677044:0:206639:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1786177433.677045:0:206639:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177433.677047:0:206639:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177433.677079:0:206639:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177433.678731:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177433.678739:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177433.678743:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177434.060585:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177434.060597:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177434.060599:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177434.060607:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177434.060611:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177434.060616:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177434.060621:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177434.060625:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177434.060628:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177434.508501:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177434.508509:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177434.508512:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177434.508518:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177434.508523:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177434.508527:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177434.508530:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177434.508532:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177434.508536:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177434.700490:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177434.700497:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177434.700503:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177434.700504:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177434.700507:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177434.700509:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177434.700589:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177434.700593:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177434.700596:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177435.084505:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177435.084513:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177435.084516:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177435.084522:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177435.084525:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177435.084530:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177435.084532:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177435.084535:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177435.084538:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177435.533319:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177435.533327:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177435.533329:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177435.533335:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177435.533339:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177435.533344:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177435.533346:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177435.533350:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177435.533353:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177435.660476:0:206639:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9800 (cpu 0) has reached its deadline 4306782468, at 4306782528 00000100:00000010:0.0:1786177435.660493:0:206639:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5ae0. 00000100:00000040:1.0:1786177435.662468:0:206600:0:(connection.c:75:cpu_latency_work()) work item of ffff9b9e04ed9910 (cpu 1) has reached its deadline 4306782468, at 4306782530 00000100:00000010:1.0:1786177435.662481:0:206600:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9b9e150d5f60. 00010000:00000001:0.0:1786177435.725066:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177435.725073:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177435.725078:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177435.725080:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177435.725083:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177435.725084:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177435.725188:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177435.725192:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177435.725202:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177436.108690:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177436.108698:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177436.108701:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177436.108707:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177436.108710:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177436.108714:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177436.108716:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177436.108718:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177436.108722:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177436.557590:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177436.557598:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177436.557601:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177436.557606:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177436.557609:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177436.557613:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177436.557615:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177436.557617:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177436.557621:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177436.748650:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177436.748657:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177436.748663:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177436.748664:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177436.748667:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177436.748668:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177436.748842:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177436.748846:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177436.748849:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1786177436.876498:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) d3dda872-7662-49cd-8c2a-7de566018653->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177436.876509:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1786177436.876512:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177436.876516:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177436.876520:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177436.876526:0:40394:0:(import.c:510:import_select_connection()) MGC192.168.202.132@tcp: connect to NID 192.168.202.132@tcp last attempt 12111 00000400:00000200:3.0:1786177436.876534:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177436.876541:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177436.876545:0:40394:0:(import.c:589:import_select_connection()) MGC192.168.202.132@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177436.876549:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.876552:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177436.876555:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.876557:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177436.876561:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177436.876564:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a1c 00000020:00000001:3.0:1786177436.876566:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177436.876570:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e10252800 refcount=3 00000020:00000001:3.0:1786177436.876572:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000020:00000001:3.0:1786177436.876575:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701910849536 : -110371798702080 : ffff9b9e10252800) 00000100:00000040:3.0:1786177436.876578:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.876580:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177436.876582:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.876584:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177436.876587:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e10252800 : new refcount 2 00000100:00080000:3.0:1786177436.876591:0:40394:0:(import.c:624:import_select_connection()) MGC192.168.202.132@tcp: import ffff9b9e07969800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177436.876593:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177436.876595:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177436.876599:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177436.876602:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.876604:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1786177436.876606:0:40394:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177436.876615:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325500. 00000020:00000040:3.0:1786177436.876619:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e07969800 refcount=4 obd=MGC192.168.202.132@tcp 02000000:00000001:3.0:1786177436.876622:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177436.876624:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.876628:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177436.876630:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177436.876634:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177436.876641:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251ab000. 00000100:00000001:3.0:1786177436.876648:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177436.876657:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177436.876668:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177436.876672:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177436.876677:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2435395] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.876734:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177436.876738:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177436.876740:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177436.876746:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.876755:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177436.876757:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177436.876761:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177436.876763:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177436.876766:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177436.876769:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235617280:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177436.876772:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177436.876777:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177436.876780:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177436.876782:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177436.876788:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3800. 02000000:00000001:0.0:1786177436.876790:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177436.876795:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177436.876799:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e4d0. 00000400:00000010:0.0:1786177436.876806:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686b28. 00000100:00000200:0.0:1786177436.876814:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235617280, portal 25 00000100:00000001:0.0:1786177436.876817:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177436.876819:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280811776 : -110371428739840 : ffff9b9e26325500) 00000100:00000040:0.0:1786177436.876824:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.876832:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177436.876834:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177436.876837:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686908. 00000100:00000200:0.0:1786177436.876840:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1872930235617280, offset 0 00000400:00000200:0.0:1786177436.876846:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177436.876853:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177436.876860:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177436.876866:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177436.876870:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177436.876874:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177436.876877:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177436.876880:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426889:426889:256:4294967295] 192.168.202.132@tcp LPNI seq info [426851:426851:8:4294967295] 00000400:00000200:0.0:1786177436.876885:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177436.876894:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177436.876901:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177436.876906:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cb00. 00000800:00000200:0.0:1786177436.876914:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177436.876918:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177436.876943:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177436.876947:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177436.876949:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 0 is 4306787584 00000100:00000010:0.0:1786177436.876954:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5ae0. 00000100:00000001:0.0:1786177436.876961:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1786177436.876969:0:2392:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS new deadline estimation for cpu 1 is 4306787584 00000100:00000010:0.0:1786177436.876972:0:2392:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9b9e150d5de0. 00000100:00000001:0.0:1786177436.876974:0:2392:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.202.132@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1786177436.876978:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177436.876981:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.876984:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.876986:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177436.879205:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177436.879213:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177436.879217:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1786177436.879218:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177436.879221:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177436.879224:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177436.879230:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12111 00000400:00000200:3.0:1786177436.879235:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177436.879239:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177436.879241:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177436.879244:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.879247:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177436.879249:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.879250:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177436.879253:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177436.879254:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a70 00000020:00000001:3.0:1786177436.879256:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177436.879258:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a18800 refcount=5 00000020:00000001:3.0:1786177436.879260:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000020:00000001:3.0:1786177436.879262:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751228416 : -110371958323200 : ffff9b9e06a18800) 00000100:00000040:3.0:1786177436.879263:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.879265:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177436.879266:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.879267:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177436.879270:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a18800 : new refcount 4 00000100:00080000:3.0:1786177436.879272:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9b9e05366000: import ffff9b9e0796c000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177436.879274:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177436.879275:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177436.879280:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177436.879282:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.879283:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177436.879286:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177436.879288:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.879289:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177436.879297:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325880. 00000020:00000040:3.0:1786177436.879299:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e0796c000 refcount=12 obd=lustre-MDT0000-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177436.879302:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177436.879303:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.879305:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177436.879308:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177436.879312:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177436.879319:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251abc00. 00000100:00000001:3.0:1786177436.879325:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177436.879334:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177436.879342:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177436.879347:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177436.879351:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c03c4dcd] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.879853:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177436.879858:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1786177436.879861:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177436.879864:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177436.879866:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177436.879873:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.879881:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177436.879883:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177436.879886:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177436.879889:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177436.880525:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177436.880534:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617408:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177436.880538:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177436.880542:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177436.880544:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177436.880546:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177436.880553:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177436.880555:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177436.880561:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177436.880564:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e108. 00000400:00000010:0.0:1786177436.880572:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686880. 00000100:00000200:0.0:1786177436.880578:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235617408, portal 10 00000100:00000001:0.0:1786177436.880580:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177436.880582:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280812672 : -110371428738944 : ffff9b9e26325880) 00000100:00000040:0.0:1786177436.880590:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.880597:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177436.880599:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177436.880602:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e316866e8. 00000100:00000200:0.0:1786177436.880605:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235617408, offset 0 00000400:00000200:0.0:1786177436.880611:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177436.880619:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177436.880628:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177436.880633:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177436.880637:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177436.880641:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177436.880644:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177436.880649:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426890:426890:255:4294967295] 192.168.202.132@tcp LPNI seq info [426852:426852:7:4294967295] 00000400:00000200:0.0:1786177436.880653:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177436.880665:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177436.880671:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177436.880677:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c500. 00000800:00000200:0.0:1786177436.880684:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177436.880689:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177436.880693:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177436.880698:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: using a timeout of 2715 usecs (3 jiffies) 00000400:00000200:0.0:1786177436.880701:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177436.880704:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306783750 00000100:00000001:0.0:1786177436.880707:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306787584 00000100:00000001:0.0:1786177436.880710:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306783750 00000100:00000001:0.0:1786177436.880713:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306787584 00000100:00000001:0.0:1786177436.880715:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177436.880717:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.880721:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.880724:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177436.880729:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177436.880731:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1786177436.880733:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177436.880734:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177436.880735:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.880736:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.881043:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177436.881046:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177436.881048:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.881050:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.881425:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177436.881430:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177436.881435:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1786177436.881436:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177436.881440:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177436.881443:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177436.881448:0:40394:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12111 00000400:00000200:3.0:1786177436.881453:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177436.881457:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177436.881460:0:40394:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177436.881464:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.881467:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177436.881470:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.881472:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177436.881475:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177436.881477:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a77 00000020:00000001:3.0:1786177436.881480:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177436.881482:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e06a19800 refcount=5 00000020:00000001:3.0:1786177436.881484:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000020:00000001:3.0:1786177436.881487:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701751232512 : -110371958319104 : ffff9b9e06a19800) 00000100:00000040:3.0:1786177436.881490:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.881492:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177436.881493:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.881495:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177436.881498:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e06a19800 : new refcount 4 00000100:00080000:3.0:1786177436.881501:0:40394:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9b9e05366000: import ffff9b9e10d37800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177436.881504:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177436.881506:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177436.881510:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177436.881513:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.881515:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177436.881518:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682930 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1786177436.881520:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.881521:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177436.881526:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26327b80. 00000020:00000040:3.0:1786177436.881529:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d37800 refcount=6 obd=lustre-MDT0001-mdc-ffff9b9e05366000 02000000:00000001:3.0:1786177436.881531:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177436.881532:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.881535:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177436.881537:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177436.881540:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177436.881544:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a9c00. 00000100:00000001:3.0:1786177436.881548:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177436.881553:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177436.881559:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177436.881562:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177436.881565:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000372aab71] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.881892:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177436.881895:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177436.881897:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177436.881902:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.881910:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177436.881912:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177436.881914:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177436.881917:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177436.881919:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177436.881923:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617536:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177436.881926:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177436.881930:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177436.881931:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177436.881933:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177436.881937:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177436.881940:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177436.881944:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177436.881947:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610ed68. 00000400:00000010:0.0:1786177436.881952:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686000. 00000100:00000200:0.0:1786177436.881958:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235617536, portal 10 00000100:00000001:0.0:1786177436.881960:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177436.881962:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280821632 : -110371428729984 : ffff9b9e26327b80) 00000100:00000040:0.0:1786177436.881969:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.881976:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177436.881979:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177436.881982:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686990. 00000100:00000200:0.0:1786177436.881984:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1872930235617536, offset 0 00000400:00000200:0.0:1786177436.881992:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177436.881997:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177436.882002:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177436.882005:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177436.882009:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177436.882012:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177436.882014:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177436.882033:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426891:426891:254:4294967295] 192.168.202.132@tcp LPNI seq info [426853:426853:6:4294967295] 00000400:00000200:0.0:1786177436.882037:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177436.882045:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177436.882050:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177436.882055:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51cf00. 00000800:00000200:0.0:1786177436.882060:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177436.882065:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177436.882070:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177436.882074:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: using a timeout of 2533 usecs (3 jiffies) 00000400:00000200:0.0:1786177436.882077:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177436.882079:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306783752 00000100:00000001:0.0:1786177436.882082:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306787584 00000100:00000001:0.0:1786177436.882084:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306783752 00000100:00000001:0.0:1786177436.882086:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306787584 00000100:00000001:0.0:1786177436.882088:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177436.882090:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.882093:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.882095:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177436.882180:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177436.882184:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177436.882187:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1786177436.882188:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177436.882190:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177436.882193:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177436.882196:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12111 00000400:00000200:3.0:1786177436.882200:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177436.882204:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177436.882206:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177436.882208:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.882211:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177436.882213:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.882216:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177436.882218:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177436.882221:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a85 00000020:00000001:3.0:1786177436.882223:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177436.882226:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08770000 refcount=4 00000020:00000001:3.0:1786177436.882228:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000020:00000001:3.0:1786177436.882230:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701781995520 : -110371927556096 : ffff9b9e08770000) 00000100:00000040:3.0:1786177436.882233:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.882235:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177436.882237:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.882239:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177436.882242:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08770000 : new refcount 3 00000100:00080000:3.0:1786177436.882245:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9b9e05366000: import ffff9b9e10d36800 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177436.882246:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177436.882248:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177436.882251:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177436.882253:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.882255:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177436.882257:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177436.882260:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.882261:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177436.882265:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26326300. 00000020:00000040:3.0:1786177436.882268:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d36800 refcount=4 obd=lustre-OST0000-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177436.882271:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177436.882272:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.882274:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177436.882275:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177436.882277:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177436.882281:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251aac00. 00000100:00000001:3.0:1786177436.882284:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177436.882290:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177436.882297:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177436.882301:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177436.882504:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000051e6ba18] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.882597:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177436.882600:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1786177436.882603:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177436.882604:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177436.882606:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177436.882611:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.882618:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177436.882619:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177436.882621:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177436.882623:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177436.882625:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177436.882629:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617664:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177436.882632:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177436.882634:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177436.882635:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177436.882637:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177436.882642:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177436.882644:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177436.882649:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177436.882652:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e688. 00000400:00000010:0.0:1786177436.882657:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686c38. 00000100:00000200:0.0:1786177436.882661:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235617664, portal 4 00000100:00000001:0.0:1786177436.882663:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177436.882665:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280815360 : -110371428736256 : ffff9b9e26326300) 00000100:00000040:0.0:1786177436.882670:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.882677:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177436.882679:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177436.882682:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686660. 00000100:00000200:0.0:1786177436.882684:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235617664, offset 0 00000400:00000200:0.0:1786177436.882689:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177436.882693:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177436.882698:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177436.882701:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177436.882702:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177436.882704:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177436.882705:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177436.882708:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426892:426892:253:4294967295] 192.168.202.132@tcp LPNI seq info [426854:426854:5:4294967295] 00000400:00000200:0.0:1786177436.882710:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177436.882715:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177436.882718:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177436.882721:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51ce00. 00000800:00000200:0.0:1786177436.882724:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177436.882726:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177436.882728:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177436.882731:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: using a timeout of 2913 usecs (3 jiffies) 00000400:00000200:0.0:1786177436.882732:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177436.882734:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306783753 00000100:00000001:0.0:1786177436.882736:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306787584 00000100:00000001:0.0:1786177436.882737:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306783753 00000100:00000001:0.0:1786177436.882739:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306787584 00000100:00000001:0.0:1786177436.882740:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177436.882741:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.882743:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.882744:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177436.883033:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177436.883036:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1786177436.883038:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177436.883040:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177436.883042:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.883044:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.883126:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177436.883127:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177436.883230:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.883234:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.883378:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1786177436.883382:0:40394:0:(pinger.c:229:ptlrpc_pinger_process_import()) 890a5a1f-bb0c-4840-beb1-f7762fa36388->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1786177436.883386:0:40394:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1786177436.883388:0:40394:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1786177436.883390:0:40394:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1786177436.883393:0:40394:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1786177436.883397:0:40394:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: connect to NID 192.168.202.132@tcp last attempt 12111 00000400:00000200:3.0:1786177436.883402:0:40394:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:3.0:1786177436.883405:0:40394:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.202.132@tcp is discovered: rc = 1 00000100:00080000:3.0:1786177436.883407:0:40394:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1786177436.883409:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.883411:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177436.883413:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.883416:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000001:3.0:1786177436.883418:0:40394:0:(genops.c:813:class_conn2export()) Process entered 00000020:00000040:3.0:1786177436.883420:0:40394:0:(genops.c:825:class_conn2export()) looking for export cookie 0x94743bada1e01a8c 00000020:00000001:3.0:1786177436.883422:0:40394:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1786177436.883424:0:40394:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9b9e08774000 refcount=4 00000020:00000001:3.0:1786177436.883426:0:40394:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000020:00000001:3.0:1786177436.883428:0:40394:0:(genops.c:827:class_conn2export()) Process leaving (rc=18446633701782011904 : -110371927539712 : ffff9b9e08774000) 00000100:00000040:3.0:1786177436.883430:0:40394:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9b9e10052880 refcount 14 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.883432:0:40394:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1786177436.883434:0:40394:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9b9e10052880 refcount 15 to 192.168.202.132@tcp 00000100:00000001:3.0:1786177436.883435:0:40394:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446633701908752512 : -110371800799104 : ffff9b9e10052880) 00000020:00000040:3.0:1786177436.883437:0:40394:0:(genops.c:900:class_export_put()) PUTting export ffff9b9e08774000 : new refcount 3 00000100:00080000:3.0:1786177436.883440:0:40394:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9b9e05366000: import ffff9b9e10d34000 using connection 192.168.202.132@tcp/192.168.202.132@tcp 00000100:00000001:3.0:1786177436.883442:0:40394:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1786177436.883445:0:40394:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1786177436.883449:0:40394:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1786177436.883451:0:40394:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.883453:0:40394:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1786177436.883457:0:40394:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682930 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1786177436.883459:0:40394:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.883461:0:40394:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1786177436.883466:0:40394:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9b9e26325c00. 00000020:00000040:3.0:1786177436.883469:0:40394:0:(genops.c:1231:class_import_get()) import ffff9b9e10d34000 refcount=3 obd=lustre-OST0001-osc-ffff9b9e05366000 02000000:00000001:3.0:1786177436.883472:0:40394:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1786177436.883473:0:40394:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1786177436.883475:0:40394:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1786177436.883477:0:40394:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1786177436.883480:0:40394:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1786177436.883485:0:40394:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9b9e251a8800. 00000100:00000001:3.0:1786177436.883489:0:40394:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1786177436.883494:0:40394:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1786177436.883502:0:40394:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1786177436.883505:0:40394:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177436.883508:0:40394:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bdc3a032] to pc [ptlrpcd_rcv+-1] req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.883551:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177436.883554:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177436.883555:0:2392:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1786177436.883560:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.883567:0:2392:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1786177436.883568:0:2392:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177436.883570:0:2392:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1786177436.883572:0:2392:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1786177436.883575:0:2392:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1786177436.883578:0:2392:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617792:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177436.883581:0:2392:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1786177436.883583:0:2392:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1786177436.883585:0:2392:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1786177436.883587:0:2392:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1786177436.883591:0:2392:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9b9e319e3000. 02000000:00000001:0.0:1786177436.883592:0:2392:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1786177436.883596:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000010:0.0:1786177436.883599:0:2392:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9b9e2610e058. 00000400:00000010:0.0:1786177436.883604:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686cc0. 00000100:00000200:0.0:1786177436.883609:0:2392:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1872930235617792, portal 4 00000100:00000001:0.0:1786177436.883611:0:2392:0:(client.c:3262:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1786177436.883613:0:2392:0:(client.c:3264:ptlrpc_request_addref()) Process leaving (rc=18446633702280813568 : -110371428738048 : ffff9b9e26325c00) 00000100:00000040:0.0:1786177436.883619:0:2392:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177436.883626:0:2392:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1786177436.883628:0:2392:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.202.132@tcp 00000400:00000010:0.0:1786177436.883631:0:2392:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9b9e31686770. 00000100:00000200:0.0:1786177436.883633:0:2392:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1872930235617792, offset 0 00000400:00000200:0.0:1786177436.883638:0:2392:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.202.132@tcp 00000400:00000200:0.0:1786177436.883643:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177436.883648:0:2392:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.132@tcp local destination 00000400:00000200:0.0:1786177436.883651:0:2392:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1786177436.883654:0:2392:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.202.32@tcp 00000400:00000200:0.0:1786177436.883657:0:2392:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.202.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1786177436.883660:0:2392:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.132@tcp 00000400:00000200:0.0:1786177436.883663:0:2392:0:(lib-move.c:1745:lnet_handle_send()) 192.168.202.32@tcp NI seq info: [426893:426893:252:4294967295] 192.168.202.132@tcp LPNI seq info [426855:426855:4:4294967295] 00000400:00000200:0.0:1786177436.883666:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:0.0:1786177436.883674:0:2392:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.202.32@tcp(192.168.202.32@tcp:) -> 192.168.202.132@tcp(192.168.202.132@tcp:192.168.202.132@tcp) : PUT try# 0 00000800:00000200:0.0:1786177436.883679:0:2392:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.132@tcp 00000800:00000010:0.0:1786177436.883684:0:2392:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9b9e1e51c200. 00000800:00000200:0.0:1786177436.883688:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000800:00000200:0.1:1786177436.883692:0:2392:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9b9e189d6d80] -> 12345-192.168.202.132@tcp (2) 00000100:00000001:0.0:1786177436.883695:0:2392:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1786177436.883698:0:2392:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: using a timeout of 4040 usecs (5 jiffies) 00000400:00000200:0.0:1786177436.883701:0:2392:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000100:00000001:0.0:1786177436.883703:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 0 is 4306783756 00000100:00000001:0.0:1786177436.883706:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 0, keep current deadline 4306787584 00000100:00000001:0.0:1786177436.883708:0:2392:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS new deadline estimation for cpu 1 is 4306783756 00000100:00000001:0.0:1786177436.883710:0:2392:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9b9e05366000: PM QoS request active for cpu 1, keep current deadline 4306787584 00000100:00000001:0.0:1786177436.883712:0:2392:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1786177436.883714:0:2392:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.883716:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.883718:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1786177436.883736:0:40394:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1786177436.883741:0:40394:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (12121) 00000100:00000001:0.0:1786177436.883779:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177436.883782:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1786177436.883784:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177436.883785:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177436.883787:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177436.883789:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177437.132529:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177437.132538:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177437.132540:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177437.132547:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177437.132551:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177437.132555:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177437.132559:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177437.132562:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177437.132565:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177437.581669:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177437.581677:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177437.581679:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177437.581685:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177437.581688:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177437.581692:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177437.581695:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177437.581697:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177437.581700:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177437.772635:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177437.772642:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177437.772648:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177437.772649:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177437.772652:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177437.772653:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177437.772961:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177437.772965:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177437.772969:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177438.157045:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177438.157062:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177438.157064:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177438.157070:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177438.157074:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177438.157078:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177438.157081:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177438.157083:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177438.157086:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177438.604673:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177438.604680:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177438.604682:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177438.604688:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177438.604691:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177438.604695:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177438.604698:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177438.604700:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177438.604703:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177438.796666:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177438.796672:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177438.796678:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177438.796679:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177438.796682:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177438.796683:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1786177438.799741:0:201810:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1786177438.799748:0:201810:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1786177438.799752:0:201810:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1786177439.180654:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177439.180670:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177439.180672:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177439.180678:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177439.180681:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177439.180685:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177439.180687:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177439.180690:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177439.180693:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177439.628528:0:206114:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177439.628540:0:206114:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177439.628543:0:206114:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177439.628549:0:206114:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e3195c180 going to sleep for 0 seconds 00000100:00000001:1.0:1786177439.628555:0:206114:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177439.628560:0:206114:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177439.628563:0:206114:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177439.628566:0:206114:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177439.628570:0:206114:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177439.820485:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177439.820491:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177439.820497:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177439.820499:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1786177439.820502:0:206639:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1786177439.820504:0:206639:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1786177439.820675:0:205873:0:(genops.c:1916:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1786177439.820679:0:205873:0:(genops.c:1930:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1786177439.820682:0:205873:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1786177439.948961:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.132:988 00000400:00000100:1.0:1786177439.949010:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.202.132@tcp at host 192.168.202.132 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1786177439.949020:0:2383:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177439.949024:0:2383:0:(router.c:1791:lnet_notify()) 192.168.202.32@tcp notifying 192.168.202.132@tcp: down 00000400:00000200:1.0:1786177439.949030:0:2383:0:(api-ni.c:1612:lnet_nid4_cpt_hash()) Match nid 192.168.202.132@tcp to cpt 0 00000400:00000200:1.0:1786177439.949034:0:2383:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1786177439.949039:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:1.0:1786177439.949043:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cb00. 00000400:00000200:1.0:1786177439.949048:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1786177439.949052:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1786177439.949058:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686908 00000400:00000010:1.0:1786177439.949061:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686908. 00000100:00000001:1.0:1786177439.949065:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177439.949073:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.949136:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177439.949140:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177439.949145:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1786177439.949163:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177439.949168:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177439.949198:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177436/real 1786177439] req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177439.949210:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177439.949212:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177439.949216:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e4d0. 00000400:00000200:0.0:1786177439.949220:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686b28 00000400:00000010:0.0:1786177439.949222:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686b28. 00000100:00000001:0.0:1786177439.949225:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177439.949228:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177439.949238:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.949245:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177439.949247:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.949250:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177439.949252:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177439.949257:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.949265:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177439.949270:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.949279:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177439.949282:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.949284:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177439.949285:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.949288:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177439.949291:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177439.949295:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e07969800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177439.949299:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177439.949302:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177439.949307:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177439.949316:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d3dda872-7662-49cd-8c2a-7de566018653:2392:1872930235617280:192.168.202.132@tcp:250:kworker.0 00000100:00000001:0.0:1786177439.949321:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177439.949323:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177439.949327:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325500 x1872930235617280/t0(0) o250->MGC192.168.202.132@tcp@192.168.202.132@tcp:26/25 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.949335:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177439.949338:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177439.949340:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177439.949341:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177439.949343:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177439.949345:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177439.949520:0:2383:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177439.949523:0:2383:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177439.949526:0:2383:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3800. 02000000:00000001:1.0:1786177439.949531:0:2383:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177439.949533:0:2383:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177439.949535:0:2383:0:(genops.c:1244:class_import_put()) import ffff9b9e07969800 refcount=3 obd=MGC192.168.202.132@tcp 00000020:00000001:1.0:1786177439.949537:0:2383:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177439.949539:0:2383:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251ab000. 02000000:00000001:1.0:1786177439.949543:0:2383:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177439.949544:0:2383:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177439.949546:0:2383:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325500. 00000100:00000001:1.0:1786177439.949549:0:2383:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177439.949551:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177439.949552:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1786177439.949562:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:1.0:1786177439.949565:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c500. 00000400:00000200:1.0:1786177439.949568:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1786177439.949572:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1786177439.949575:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e316866e8 00000400:00000010:1.0:1786177439.949577:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e316866e8. 00000100:00000001:1.0:1786177439.949580:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177439.949589:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177439.949606:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177439.949610:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177439.949616:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177439.949617:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1786177439.949622:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:1.0:1786177439.949625:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51cf00. 00000400:00000200:1.0:1786177439.949627:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1786177439.949630:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1786177439.949632:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686990 00000400:00000010:1.0:1786177439.949634:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686990. 00000100:00000001:1.0:1786177439.949636:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177439.949639:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177439.949646:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177439.949649:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177439.949655:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177439.949657:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1786177439.949661:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:1.0:1786177439.949663:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51ce00. 00000400:00000200:1.0:1786177439.949665:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1786177439.949668:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1786177439.949670:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686660 00000400:00000010:1.0:1786177439.949672:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686660. 00000100:00000001:1.0:1786177439.949674:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177439.949677:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177439.949683:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1786177439.949686:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177439.949691:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1786177439.949693:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1786177439.949696:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.132@tcp 00000800:00000010:1.0:1786177439.949698:0:2383:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9b9e1e51c200. 00000400:00000200:1.0:1786177439.949700:0:2383:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1786177439.949703:0:2383:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.202.32@tcp->192.168.202.132@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1786177439.949705:0:2383:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686770 00000400:00000010:1.0:1786177439.949707:0:2383:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686770. 00000100:00000001:1.0:1786177439.949708:0:2383:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1786177439.949712:0:2383:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177439.949719:0:2383:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1786177439.949845:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177439.949847:0:2392:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1786177439.949848:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177439.949853:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177436/real 1786177439] req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177439.949862:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177439.949863:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177439.949865:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e058. 00000400:00000200:0.0:1786177439.949867:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686cc0 00000400:00000010:0.0:1786177439.949869:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686cc0. 00000100:00000001:0.0:1786177439.949872:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177439.949875:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177439.949888:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.949895:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177439.949896:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.949898:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177439.949899:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177439.949902:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.949908:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177439.949911:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.949917:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177439.949919:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.949921:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177439.949922:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.949924:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177439.949926:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177439.949928:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d34000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177439.949932:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177439.949934:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177439.949938:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177439.949946:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617792:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177439.949949:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177439.949953:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177436/real 1786177439] req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177439.949966:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177439.949968:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177439.949970:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e688. 00000400:00000200:0.0:1786177439.949972:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686c38 00000400:00000010:0.0:1786177439.949974:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686c38. 00000100:00000001:0.0:1786177439.949976:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177439.949980:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177439.949997:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950003:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177439.950005:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950007:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177439.950008:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177439.950011:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950018:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177439.950021:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950028:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177439.950030:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950032:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177439.950033:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950035:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177439.950036:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177439.950038:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d36800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177439.950041:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177439.950043:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177439.950048:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177439.950055:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26326300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617664:192.168.202.132@tcp:8:kworker.0 00000100:00000001:0.0:1786177439.950059:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177439.950062:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177436/real 1786177439] req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177439.950069:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177439.950070:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177439.950072:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610ed68. 00000400:00000200:0.0:1786177439.950075:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686000 00000400:00000010:0.0:1786177439.950076:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686000. 00000100:00000001:0.0:1786177439.950078:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177439.950081:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177439.950090:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950096:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177439.950097:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950098:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177439.950099:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177439.950103:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950108:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177439.950112:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950117:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177439.950119:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950121:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177439.950123:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950124:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177439.950126:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177439.950129:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e10d37800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177439.950132:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177439.950134:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177439.950137:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177439.950144:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26327b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617536:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177439.950147:0:2392:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1786177439.950151:0:2392:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1786177436/real 1786177439] req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1786177439.950157:0:2392:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1786177439.950159:0:2392:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1786177439.950160:0:2392:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9b9e2610e108. 00000400:00000200:0.0:1786177439.950162:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9b9e31686880 00000400:00000010:0.0:1786177439.950164:0:2392:0:(lib-lnet.h:277:lnet_md_free()) slab-freed 'md': 136 at ffff9b9e31686880. 00000100:00000001:0.0:1786177439.950166:0:2392:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1786177439.950168:0:2392:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1786177439.950176:0:2392:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950182:0:2392:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1786177439.950183:0:2392:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950185:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177439.950186:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1786177439.950190:0:2392:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950201:0:2392:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1786177439.950204:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950211:0:2392:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1786177439.950213:0:2392:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950215:0:2392:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1786177439.950216:0:2392:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950217:0:2392:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1786177439.950220:0:2392:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1786177439.950222:0:2392:0:(import.c:65:import_set_state_nolock()) ffff9b9e0796c000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1786177439.950226:0:2392:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.132@tcp failed (-110) 00000100:00000001:0.0:1786177439.950228:0:2392:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1786177439.950231:0:2392:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1786177439.950238:0:2392:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9b9e26325880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:890a5a1f-bb0c-4840-beb1-f7762fa36388:2392:1872930235617408:192.168.202.132@tcp:38:kworker.0 00000100:00000001:0.0:1786177439.950242:0:2392:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950244:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177439.950247:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 2 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950254:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177439.950255:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177439.950258:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26326300 x1872930235617664/t0(0) o8->lustre-OST0000-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950263:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177439.950265:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177439.950267:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e0000. 02000000:00000001:0.0:1786177439.950270:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177439.950272:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177439.950273:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d36800 refcount=3 obd=lustre-OST0000-osc-ffff9b9e05366000 00000020:00000001:0.0:1786177439.950275:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177439.950276:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251aac00. 02000000:00000001:0.0:1786177439.950279:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177439.950280:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177439.950281:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26326300. 00000100:00000001:0.0:1786177439.950284:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177439.950285:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950286:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177439.950289:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26327b80 x1872930235617536/t0(0) o38->lustre-MDT0001-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950294:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177439.950295:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177439.950297:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e1800. 02000000:00000001:0.0:1786177439.950298:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177439.950299:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177439.950300:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e10d37800 refcount=5 obd=lustre-MDT0001-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177439.950302:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177439.950304:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a9c00. 02000000:00000001:0.0:1786177439.950306:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177439.950307:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177439.950308:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26327b80. 00000100:00000001:0.0:1786177439.950310:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177439.950311:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950313:0:2392:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1786177439.950316:0:2392:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9b9e26325880 x1872930235617408/t0(0) o38->lustre-MDT0000-mdc-ffff9b9e05366000@192.168.202.132@tcp:12/10 lens 520/544 e 0 to 1 dl 1786177461 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1786177439.950322:0:2392:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1786177439.950324:0:2392:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1786177439.950325:0:2392:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e2000. 02000000:00000001:0.0:1786177439.950327:0:2392:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1786177439.950328:0:2392:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:0.0:1786177439.950329:0:2392:0:(genops.c:1244:class_import_put()) import ffff9b9e0796c000 refcount=11 obd=lustre-MDT0000-mdc-ffff9b9e05366000 00000020:00000001:0.0:1786177439.950331:0:2392:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:0.0:1786177439.950332:0:2392:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251abc00. 02000000:00000001:0.0:1786177439.950334:0:2392:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1786177439.950335:0:2392:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1786177439.950336:0:2392:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325880. 00000100:00000001:0.0:1786177439.950338:0:2392:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1786177439.950339:0:2392:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950341:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1786177439.950497:0:2392:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1786177439.950500:0:2392:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1786177439.950503:0:2392:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1786177439.950505:0:2392:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1786177439.951830:0:2383:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9b9e26325c00 x1872930235617792/t0(0) o8->lustre-OST0001-osc-ffff9b9e05366000@192.168.202.132@tcp:28/4 lens 520/544 e 0 to 0 dl 1786177461 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1786177439.951841:0:2383:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1786177439.951844:0:2383:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1786177439.951846:0:2383:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9b9e319e3000. 02000000:00000001:1.0:1786177439.951851:0:2383:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1786177439.951852:0:2383:0:(genops.c:1238:class_import_put()) Process entered 00000020:00000040:1.0:1786177439.951854:0:2383:0:(genops.c:1244:class_import_put()) import ffff9b9e10d34000 refcount=2 obd=lustre-OST0001-osc-ffff9b9e05366000 00000020:00000001:1.0:1786177439.951856:0:2383:0:(genops.c:1251:class_import_put()) Process leaving 02000000:00000010:1.0:1786177439.951859:0:2383:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9b9e251a8800. 02000000:00000001:1.0:1786177439.951862:0:2383:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1786177439.951864:0:2383:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1786177439.951865:0:2383:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9b9e26325c00. 00000100:00000001:1.0:1786177439.951868:0:2383:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1786177439.951870:0:2383:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177439.951872:0:2383:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1786177440.206847:0:201811:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1786177440.206855:0:201811:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1786177440.206857:0:201811:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1786177440.206862:0:201811:0:(client.c:2702:ptlrpc_set_wait()) set ffff9b9e10245f00 going to sleep for 0 seconds 00000100:00000001:1.0:1786177440.206866:0:201811:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1786177440.206870:0:201811:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177440.206872:0:201811:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1786177440.206875:0:201811:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1786177440.206878:0:201811:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) Debug log: 137129 lines, 137129 kept, 0 dropped, 0 bad.